build: Centralize our protobuf compilation steps
We have a few different .proto files in this repository that all need to get recompiled into .pb.go files each time we change them, but we were previously handling that with some scripts that just assumed that protoc and the relevant plugins were already installed on the system somewhere, at the right versions. In practice we've been constantly flopping between different versions of these tools due to folks having different versions installed in their development environments. In particular, the state of the .pb.go files in the prior commit wasn't reproducible by any single version of the tools because they've all slightly diverged from one another. In the interests of being more consistent here and avoiding accidental inconsistencies, we'll now centralize the protocol buffer compile steps all into a single tool that knows how to fetch and install the expected versions of the various tools we need and then run those tools with the right options to get a stable result. If we want to upgrade to either a newer protoc or a newer protoc-gen-go in future then we'll do that in a central location and update all of the .pb.go files at the same time, so that we're always consistently tracking the same version of protocol buffers everywhere. While doing this I attempted to keep as close as possible to the toolchain we'd most recently used, but since they were not consistent with each other they've now all changed which version numbers they record at minimum, and the planproto stub in particular now also has a slightly different descriptor serialization but is otherwise offering the same API.
This commit is contained in:
parent
249e05f827
commit
ce96d82de0
9
Makefile
9
Makefile
|
@ -11,12 +11,11 @@ generate:
|
|||
# Terraform do not involve changing protobuf files and protoc is not a
|
||||
# go-gettable dependency and so getting it installed can be inconvenient.
|
||||
#
|
||||
# If you are working on changes to protobuf interfaces you may either use
|
||||
# this target or run the individual scripts below directly.
|
||||
# If you are working on changes to protobuf interfaces, run this Makefile
|
||||
# target to be sure to regenerate all of the protobuf stubs using the expected
|
||||
# versions of protoc and the protoc Go plugins.
|
||||
protobuf:
|
||||
bash scripts/protobuf-check.sh
|
||||
bash internal/tfplugin5/generate.sh
|
||||
bash internal/plans/internal/planproto/generate.sh
|
||||
go run ./tools/protobuf-compile .
|
||||
|
||||
fmtcheck:
|
||||
@sh -c "'$(CURDIR)/scripts/gofmtcheck.sh'"
|
||||
|
|
1
go.mod
1
go.mod
|
@ -166,6 +166,7 @@ require (
|
|||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 // indirect
|
||||
google.golang.org/grpc v1.36.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
|
||||
google.golang.org/protobuf v1.25.0
|
||||
gopkg.in/inf.v0 v0.9.0 // indirect
|
||||
gopkg.in/ini.v1 v1.42.0 // indirect
|
||||
|
|
2
go.sum
2
go.sum
|
@ -1011,6 +1011,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM
|
|||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
|
||||
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 h1:M1YKkFIboKNieVO5DLUEVzQfGwJD30Nv2jfUgzb5UcE=
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
|
|
@ -1,16 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# We do not run protoc under go:generate because we want to ensure that all
|
||||
# dependencies of go:generate are "go get"-able for general dev environment
|
||||
# usability. To compile all protobuf files in this repository, run
|
||||
# "make protobuf" at the top-level.
|
||||
|
||||
set -eu
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
cd "$DIR"
|
||||
|
||||
protoc --go_out=paths=source_relative:. planfile.proto
|
|
@ -1,12 +1,13 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.26.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.15.6
|
||||
// source: planfile.proto
|
||||
|
||||
package planproto
|
||||
|
||||
import (
|
||||
proto "github.com/golang/protobuf/proto"
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
reflect "reflect"
|
||||
|
@ -20,6 +21,10 @@ const (
|
|||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
// This is a compile-time assertion that a sufficiently up-to-date version
|
||||
// of the legacy proto package is being used.
|
||||
const _ = proto.ProtoPackageIsVersion4
|
||||
|
||||
// Mode describes the planning mode that created the plan.
|
||||
type Mode int32
|
||||
|
||||
|
@ -1192,11 +1197,12 @@ var file_planfile_proto_rawDesc = []byte{
|
|||
0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x59,
|
||||
0x5f, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x02, 0x12, 0x21, 0x0a, 0x1d, 0x52, 0x45,
|
||||
0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x45, 0x43, 0x41, 0x55, 0x53, 0x45, 0x5f, 0x43, 0x41,
|
||||
0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x42, 0x39, 0x5a,
|
||||
0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68,
|
||||
0x4e, 0x4e, 0x4f, 0x54, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x03, 0x42, 0x42, 0x5a,
|
||||
0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68,
|
||||
0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x74, 0x65, 0x72, 0x72, 0x61, 0x66, 0x6f, 0x72, 0x6d, 0x2f,
|
||||
0x70, 0x6c, 0x61, 0x6e, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70,
|
||||
0x6c, 0x61, 0x6e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x73, 0x2f, 0x69,
|
||||
0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x6c, 0x61, 0x6e, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
|
|
@ -1,16 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# We do not run protoc under go:generate because we want to ensure that all
|
||||
# dependencies of go:generate are "go get"-able for general dev environment
|
||||
# usability. To compile all protobuf files in this repository, run
|
||||
# "make protobuf" at the top-level.
|
||||
|
||||
set -eu
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
cd "$DIR"
|
||||
|
||||
protoc --go_out=paths=source_relative,plugins=grpc:. ./tfplugin5.proto
|
|
@ -20,7 +20,7 @@
|
|||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.14.0
|
||||
// protoc v3.15.6
|
||||
// source: tfplugin5.proto
|
||||
|
||||
package tfplugin5
|
||||
|
|
|
@ -1,16 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# We do not run protoc under go:generate because we want to ensure that all
|
||||
# dependencies of go:generate are "go get"-able for general dev environment
|
||||
# usability. To compile all protobuf files in this repository, run
|
||||
# "make protobuf" at the top-level.
|
||||
|
||||
set -eu
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
|
||||
cd "$DIR"
|
||||
|
||||
protoc --go_out=paths=source_relative,plugins=grpc:. ./tfplugin6.proto
|
|
@ -19,8 +19,8 @@
|
|||
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.23.0
|
||||
// protoc v3.13.0
|
||||
// protoc-gen-go v1.25.0
|
||||
// protoc v3.15.6
|
||||
// source: tfplugin6.proto
|
||||
|
||||
package tfplugin6
|
||||
|
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Check whether protobuf & go plugin are installed
|
||||
PROTOC_HELP_URL="http://google.github.io/proto-lens/installing-protoc.html"
|
||||
PROTOC_GEN_GO_HELP_URL="https://github.com/golang/protobuf/tree/v1.3.2#installation"
|
||||
|
||||
EXIT_CODE=0
|
||||
|
||||
which protoc >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Protocol Buffers not found."
|
||||
echo "Please install Protocol Buffers and ensure 'protoc' is available in your PATH."
|
||||
echo "See ${PROTOC_HELP_URL} for more."
|
||||
echo
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
which protoc-gen-go >/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Protocol Buffers Go plugin not found."
|
||||
echo "Please install the plugin and ensure 'protoc-gen-go' is available in your PATH."
|
||||
echo "See ${PROTOC_GEN_GO_HELP_URL} for more."
|
||||
echo
|
||||
EXIT_CODE=1
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
|
@ -0,0 +1,5 @@
|
|||
# This directory acts both as a cache so we can avoid constantly re-downloading
|
||||
# the same protoc, and as a staging area where we can put a protoc-gen-go
|
||||
# executable that won't interfere with the operation of other Go codebases
|
||||
# on the same system which might want a different version of protoc-gen-go.
|
||||
protoc-*
|
|
@ -0,0 +1,233 @@
|
|||
// protobuf-compile is a helper tool for running protoc against all of the
|
||||
// .proto files in this repository using specific versions of protoc and
|
||||
// protoc-gen-go, to ensure consistent results across all development
|
||||
// environments.
|
||||
//
|
||||
// protoc itself isn't a Go tool, so we need to use a custom strategy to
|
||||
// install and run it. The official releases are built only for a subset of
|
||||
// platforms that Go can potentially target, so this tool will fail if you
|
||||
// are using a platform other than the ones this wrapper tool has explicit
|
||||
// support for. In that case you'll need to either run this tool on a supported
|
||||
// platform or to recreate what it does manually using a protoc you've built
|
||||
// and installed yourself.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/go-getter"
|
||||
)
|
||||
|
||||
const protocVersion = "3.15.6"
|
||||
|
||||
// We also use protoc-gen-go and its grpc addon, but since these are Go tools
|
||||
// in Go modules our version selection for these comes from our top-level
|
||||
// go.mod, as with all other Go dependencies. If you want to switch to a newer
|
||||
// version of either tool then you can upgrade their modules in the usual way.
|
||||
const protocGenGoPackage = "github.com/golang/protobuf/protoc-gen-go"
|
||||
const protocGenGoGrpcPackage = "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
|
||||
|
||||
type protocStep struct {
|
||||
DisplayName string
|
||||
WorkDir string
|
||||
Args []string
|
||||
}
|
||||
|
||||
var protocSteps = []protocStep{
|
||||
{
|
||||
"tfplugin5 (provider wire protocol version 5)",
|
||||
"internal/tfplugin5",
|
||||
[]string{"--go_out=paths=source_relative,plugins=grpc:.", "./tfplugin5.proto"},
|
||||
},
|
||||
{
|
||||
"tfplugin6 (provider wire protocol version 6)",
|
||||
"internal/tfplugin6",
|
||||
[]string{"--go_out=paths=source_relative,plugins=grpc:.", "./tfplugin6.proto"},
|
||||
},
|
||||
{
|
||||
"tfplan (plan file serialization)",
|
||||
"internal/plans/internal/planproto",
|
||||
[]string{"--go_out=paths=source_relative:.", "planfile.proto"},
|
||||
},
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) != 2 {
|
||||
log.Fatal("Usage: go run github.com/hashicorp/terraform/tools/protobuf-compile <basedir>")
|
||||
}
|
||||
baseDir := os.Args[1]
|
||||
workDir := filepath.Join(baseDir, "tools/protobuf-compile/.workdir")
|
||||
|
||||
protocLocalDir := filepath.Join(workDir, "protoc-v"+protocVersion)
|
||||
if _, err := os.Stat(protocLocalDir); os.IsNotExist(err) {
|
||||
err := downloadProtoc(protocVersion, protocLocalDir)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
log.Printf("already have protoc v%s in %s", protocVersion, protocLocalDir)
|
||||
}
|
||||
|
||||
protocExec := filepath.Join(protocLocalDir, "bin/protoc")
|
||||
|
||||
protocGenGoExec, err := buildProtocGenGo(workDir)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
protocGenGoGrpcExec, err := buildProtocGenGoGrpc(workDir)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
protocExec, err = filepath.Abs(protocExec)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
protocGenGoExec, err = filepath.Abs(protocGenGoExec)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
protocGenGoGrpcExec, err = filepath.Abs(protocGenGoExec)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// For all of our steps we'll run our localized protoc with our localized
|
||||
// protoc-gen-go.
|
||||
baseCmdLine := []string{protocExec, "--plugin=" + protocGenGoExec, "--plugin=" + protocGenGoGrpcExec}
|
||||
|
||||
for _, step := range protocSteps {
|
||||
log.Printf("working on %s", step.DisplayName)
|
||||
|
||||
cmdLine := make([]string, 0, len(baseCmdLine)+len(step.Args))
|
||||
cmdLine = append(cmdLine, baseCmdLine...)
|
||||
cmdLine = append(cmdLine, step.Args...)
|
||||
|
||||
cmd := &exec.Cmd{
|
||||
Path: cmdLine[0],
|
||||
Args: cmdLine[1:],
|
||||
Dir: step.WorkDir,
|
||||
Env: os.Environ(),
|
||||
Stdout: os.Stdout,
|
||||
Stderr: os.Stderr,
|
||||
}
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
log.Printf("failed to compile: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// downloadProtoc downloads the given version of protoc into the given
|
||||
// directory.
|
||||
func downloadProtoc(version string, localDir string) error {
|
||||
protocURL, err := protocDownloadURL(version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("downloading and extracting protoc v%s from %s into %s", version, protocURL, localDir)
|
||||
|
||||
// For convenience, we'll be using go-getter to actually download this
|
||||
// thing, so we need to turn the real URL into the funny sort of pseudo-URL
|
||||
// thing that go-getter wants.
|
||||
goGetterURL := protocURL + "?archive=zip"
|
||||
|
||||
err = getter.Get(localDir, goGetterURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download or extract the package: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildProtocGenGo uses the Go toolchain to fetch the module containing
|
||||
// protoc-gen-go and then build an executable into the working directory.
|
||||
//
|
||||
// If successful, it returns the location of the executable.
|
||||
func buildProtocGenGo(workDir string) (string, error) {
|
||||
exeSuffixRaw, err := exec.Command("go", "env", "GOEXE").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to determine executable suffix: %s", err)
|
||||
}
|
||||
exeSuffix := strings.TrimSpace(string(exeSuffixRaw))
|
||||
exePath := filepath.Join(workDir, "protoc-gen-go"+exeSuffix)
|
||||
log.Printf("building %s as %s", protocGenGoPackage, exePath)
|
||||
|
||||
cmd := exec.Command("go", "build", "-o", exePath, protocGenGoPackage)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to build %s: %s", protocGenGoPackage, err)
|
||||
}
|
||||
|
||||
return exePath, nil
|
||||
}
|
||||
|
||||
// buildProtocGenGoGrpc uses the Go toolchain to fetch the module containing
|
||||
// protoc-gen-go-grpc and then build an executable into the working directory.
|
||||
//
|
||||
// If successful, it returns the location of the executable.
|
||||
func buildProtocGenGoGrpc(workDir string) (string, error) {
|
||||
exeSuffixRaw, err := exec.Command("go", "env", "GOEXE").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to determine executable suffix: %s", err)
|
||||
}
|
||||
exeSuffix := strings.TrimSpace(string(exeSuffixRaw))
|
||||
exePath := filepath.Join(workDir, "protoc-gen-go-grpc"+exeSuffix)
|
||||
log.Printf("building %s as %s", protocGenGoGrpcPackage, exePath)
|
||||
|
||||
cmd := exec.Command("go", "build", "-o", exePath, protocGenGoGrpcPackage)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to build %s: %s", protocGenGoGrpcPackage, err)
|
||||
}
|
||||
|
||||
return exePath, nil
|
||||
}
|
||||
|
||||
// protocDownloadURL returns the URL to try to download the protoc package
|
||||
// for the current platform or an error if there's no known URL for the
|
||||
// current platform.
|
||||
func protocDownloadURL(version string) (string, error) {
|
||||
platformKW := protocPlatform()
|
||||
if platformKW == "" {
|
||||
return "", fmt.Errorf("don't know where to find protoc for %s on %s", runtime.GOOS, runtime.GOARCH)
|
||||
}
|
||||
return fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-%s.zip", protocVersion, protocVersion, platformKW), nil
|
||||
}
|
||||
|
||||
// protocPlatform returns the package name substring for the current platform
|
||||
// in the naming convention used by official protoc packages, or an empty
|
||||
// string if we don't know how protoc packaging would describe current
|
||||
// platform.
|
||||
func protocPlatform() string {
|
||||
goPlatform := runtime.GOOS + "_" + runtime.GOARCH
|
||||
|
||||
switch goPlatform {
|
||||
case "linux_amd64":
|
||||
return "linux-x86_64"
|
||||
case "linux_arm64":
|
||||
return "linux-aarch_64"
|
||||
case "darwin_amd64":
|
||||
return "osx-x86_64"
|
||||
case "darwin_arm64":
|
||||
// As of 3.15.6 there isn't yet an osx-aarch_64 package available,
|
||||
// so we'll install the x86_64 version and hope Rosetta can handle it.
|
||||
return "osx-x86_64"
|
||||
case "windows_amd64":
|
||||
return "win64" // for some reason the windows packages don't have a CPU architecture part
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
|
@ -5,7 +5,9 @@ package tools
|
|||
|
||||
import (
|
||||
_ "github.com/golang/mock/mockgen"
|
||||
_ "github.com/golang/protobuf/protoc-gen-go"
|
||||
_ "github.com/mitchellh/gox"
|
||||
_ "golang.org/x/tools/cmd/cover"
|
||||
_ "golang.org/x/tools/cmd/stringer"
|
||||
_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
|
||||
)
|
||||
|
|
Loading…
Reference in New Issue