From c133de863b9f6cce231819c8787d66d3112fc31c Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 16 Nov 2018 18:54:33 -0800 Subject: [PATCH] build: Update most things for Go 1.11 modules We're still using vendoring for now until we get _all_ of our tooling updated, so the main idea here is to force use of the vendor directory when running tests and building for development so we can quickly find situations where we forget to run "go mod vendor". We also setting GO111MODULE=off for installation of tools. Right now this is the best way to install a tool in GOBIN without also interfering with go.mod and go.sum, until a better pattern for managing tool dependencies is devised by the Go team. Finally, we run "go mod download" before launching "gox" in the main build process, to prime the local module cache once so that the concurrent "go build" processes won't race to populate it redundantly. This means that we'll be producing final builds from the module cache rather than from vendor as with everything else -- there's currently no way to tell gox to use -mod=vendor -- but that should be fine in practice since our go.sum file will ensure that we get the exact sources we expect in the module cache before building. --- .travis.yml | 2 +- Makefile | 33 ++++++++++++++++++--------------- scripts/build.sh | 4 ++++ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 665a74e87..787a70a08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ go: # add TF_ETCDV3_TEST=1 to run etcdv3 tests # if added, TF_ETCDV3_ENDPOINTS must be set to a comma-separated list of (insecure) etcd endpoints against which to test env: - - CONSUL_VERSION=0.7.5 GOMAXPROCS=4 + - CONSUL_VERSION=0.7.5 GOMAXPROCS=4 GO111MODULE=on # Fetch consul for the backend and provider tests before_install: diff --git a/Makefile b/Makefile index 6c08f8e18..305435d67 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ WEBSITE_REPO=github.com/hashicorp/terraform-website default: test tools: - go get -u github.com/kardianos/govendor - go get -u golang.org/x/tools/cmd/stringer - go get -u golang.org/x/tools/cmd/cover - go get -u github.com/golang/mock/mockgen + GO111MODULE=off go get -u github.com/kardianos/govendor + GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer + GO111MODULE=off go get -u golang.org/x/tools/cmd/cover + GO111MODULE=off go get -u github.com/golang/mock/mockgen # bin generates the releaseable binaries for Terraform bin: fmtcheck generate @@ -18,10 +18,10 @@ bin: fmtcheck generate # dev creates binaries for testing Terraform locally. These are put # into ./bin/ as well as $GOPATH/bin dev: fmtcheck generate - @TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'" + go install -mod=vendor . quickdev: generate - @TF_DEV=1 sh -c "'$(CURDIR)/scripts/build.sh'" + go install -mod=vendor . # Shorthand for building and installing just one plugin for local testing. # Run as (for example): make plugin-dev PLUGIN=provider-aws @@ -33,34 +33,34 @@ plugin-dev: generate # we run this one package at a time here because running the entire suite in # one command creates memory usage issues when running in Travis-CI. test: fmtcheck generate - go list $(TEST) | xargs -t -n4 go test $(TESTARGS) -timeout=2m -parallel=4 + go list -mod=vendor $(TEST) | xargs -t -n4 go test $(TESTARGS) -mod=vendor -timeout=2m -parallel=4 # testacc runs acceptance tests testacc: fmtcheck generate @if [ "$(TEST)" = "./..." ]; then \ echo "ERROR: Set TEST to a specific package. For example,"; \ - echo " make testacc TEST=./builtin/providers/aws"; \ + echo " make testacc TEST=./builtin/providers/test"; \ exit 1; \ fi - TF_ACC=1 go test $(TEST) -v $(TESTARGS) -timeout 120m + TF_ACC=1 go test $(TEST) -v $(TESTARGS) -mod=vendor -timeout 120m # e2etest runs the end-to-end tests against a generated Terraform binary # The TF_ACC here allows network access, but does not require any special # credentials since the e2etests use local-only providers such as "null". e2etest: generate - TF_ACC=1 go test -v ./command/e2etest + TF_ACC=1 go test -mod=vendor -v ./command/e2etest test-compile: fmtcheck generate @if [ "$(TEST)" = "./..." ]; then \ echo "ERROR: Set TEST to a specific package. For example,"; \ - echo " make test-compile TEST=./builtin/providers/aws"; \ + echo " make test-compile TEST=./builtin/providers/test"; \ exit 1; \ fi go test -c $(TEST) $(TESTARGS) # testrace runs the race checker testrace: fmtcheck generate - TF_ACC= go test -race $(TEST) $(TESTARGS) + TF_ACC= go test -mod=vendor -race $(TEST) $(TESTARGS) cover: @go tool cover 2>/dev/null; if [ $$? -eq 3 ]; then \ @@ -75,10 +75,13 @@ cover: # "make protobuf". generate: @which stringer > /dev/null; if [ $$? -ne 0 ]; then \ - go get -u golang.org/x/tools/cmd/stringer; \ + GO111MODULE=off go get -u golang.org/x/tools/cmd/stringer; \ fi - go generate ./... - @go fmt command/internal_plugin_list.go > /dev/null + # We turn off modules for "go generate" because our downstream generate + # commands are not all ready to deal with Go modules yet, and this + # avoids downloading all of the deps that are in the vendor dir anyway. + GO111MODULE=off go generate ./... + GO111MODULE=off go fmt command/internal_plugin_list.go > /dev/null # We separate the protobuf generation because most development tasks on # Terraform do not involve changing protobuf files and protoc is not a diff --git a/scripts/build.sh b/scripts/build.sh index 45e297d1d..bdef1111f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -47,6 +47,10 @@ if [[ -n "${TF_RELEASE}" ]]; then LD_FLAGS="-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X github.com/hashicorp/terraform/version.Prerelease= -s -w" fi +# Ensure all remote modules are downloaded and cached before build so that +# the concurrent builds launched by gox won't race to redundantly download them. +go mod download + # Build! echo "==> Building..." gox \