build: Build and run e2etest as part of the release build pipeline

This uses the decoupled build and run strategy to run the e2etests so that
we can arrange to run the tests against the real release packages produced
elsewhere in this workflow, rather than ones generated just in time by
the test harness.

The modifications to make-archive.sh here make it more consistent with its
originally-intended purpose of producing a harness for testing "real"
release executables. Our earlier compromise of making it include its own
terraform executable came from a desire to use that script as part of
manual cross-platform testing when we weren't yet set up to support
automation of those tests as we're doing here. That does mean, however,
that the terraform-e2etest package content must be combined with content
from a terraform release package in order to produce a valid contest for
running the tests.

We use a single job to cross-compile the test harness for all of the
supported platforms, because that build is relatively fast and so not
worth the overhead of matrix build, but then use a matrix build to
actually run the tests so that we can run them in a worker matching the
target platform.

We currently have access only to amd64 (x64) runners in GitHub Actions
and so for the moment this process is limited only to the subset of our
supported platforms which use that architecture.

.github/workflows/build.yml

@@ -28,11 +28,22 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0 # Need all commits and tags to find a reasonable version number
-      - name: Decide version number
+      - name: Git Describe
-        id: get-product-version
+        id: git-describe
         run: |
           git describe --first-parent
-          echo "::set-output name=product-version::$(git describe --first-parent)"
+          echo "::set-output name=raw-version::$(git describe --first-parent)"
+      - name: Decide version number
+        id: get-product-version
+        shell: bash
+        env:
+          RAW_VERSION: ${{ steps.git-describe.outputs.raw-version }}
+        run: |
+          echo "::set-output name=product-version::${RAW_VERSION#v}"
+      - name: Report chosen version number
+        run: |
+          [ -n "${{steps.get-product-version.outputs.product-version}}" ]
+          echo "::notice title=Terraform CLI Version::${{ steps.get-product-version.outputs.product-version }}"
 
   get-go-version:
     name: "Determine Go toolchain version"
@@ -207,3 +218,195 @@ jobs:
           tags: |
             docker.io/hashicorp/${{env.repo}}:${{env.version}}
             986891699432.dkr.ecr.us-east-1.amazonaws.com/hashicorp/${{env.repo}}:${{env.version}}
+
+  e2etest-build:
+    name: Build e2etest for ${{ matrix.goos }}_${{ matrix.goarch }}
+    needs: ["get-go-version"]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # We build test harnesses only for the v1.0 Compatibility Promises
+        # supported platforms. Even within that set, we can only run on
+        # architectures for which we have GitHub Actions runners available,
+        # which is currently only amd64 (x64).
+        # TODO: GitHub Actions does support _self-hosted_ arm and arm64
+        # runners, so we could potentially run some ourselves to run our
+        # tests there, but at the time of writing there is no documented
+        # support for darwin_arm64 (macOS on Apple Silicon).
+        include:
+          - {goos: "darwin", goarch: "amd64"}
+          #- {goos: "darwin", goarch: "arm64"}
+          - {goos: "windows", goarch: "amd64"}
+          - {goos: "linux", goarch: "amd64"}
+          #- {goos: "linux", goarch: "arm"}
+          #- {goos: "linux", goarch: "arm64"}
+      fail-fast: false
+
+    env:
+      build_script: ./internal/command/e2etest/make-archive.sh
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+
+      - name: Build test harness package
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+        run: |
+          bash ./internal/command/e2etest/make-archive.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          path: internal/command/e2etest/build/terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          if-no-files-found: error
+
+  e2etest-linux:
+    name: e2etest for linux_${{ matrix.goarch }}
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+      - build
+      - e2etest-build
+
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+          #- {goarch: "arm64"}
+          #- {goarch: "arm"}
+      fail-fast: false
+
+    env:
+      os: linux
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        run: |
+          unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip"
+          unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip"
+      - name: Run E2E Tests
+        run: |
+          TF_ACC=1 ./e2etest -test.v
+
+  e2etest-darwin:
+    name: e2etest for darwin_${{ matrix.goarch }}
+    runs-on: macos-latest
+    needs:
+      - get-product-version
+      - build-darwin
+      - e2etest-build
+
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+          #- {goarch: "arm64"}
+      fail-fast: false
+
+    env:
+      os: darwin
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        run: |
+          unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip"
+          unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip"
+      - name: Run E2E Tests
+        run: |
+          TF_ACC=1 ./e2etest -test.v
+
+  e2etest-windows:
+    name: e2etest for windows_${{ matrix.goarch }}
+    runs-on: windows-latest
+    needs:
+      - get-product-version
+      - build
+      - e2etest-build
+
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+      fail-fast: false
+
+    env:
+      os: windows
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        shell: pwsh
+        run: |
+          Expand-Archive -LiteralPath 'terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.'
+          Expand-Archive -LiteralPath 'terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.'
+      - name: Run E2E Tests
+        env:
+          TF_ACC: 1
+        shell: cmd
+        run: |
+          e2etest.exe -test.v

internal/command/e2etest/main_test.go

@@ -11,6 +11,18 @@ import (
 
 var terraformBin string
 
+// canRunGoBuild is a short-term compromise to account for the fact that we
+// have a small number of tests that work by building helper programs using
+// "go build" at runtime, but we can't do that in our isolated test mode
+// driven by the make-archive.sh script.
+//
+// FIXME: Rework this a bit so that we build the necessary helper programs
+// (test plugins, etc) as part of the initial suite setup, and in the
+// make-archive.sh script, so that we can run all of the tests in both
+// situations with the tests just using the executable already built for
+// them, as we do for terraformBin.
+var canRunGoBuild bool
+
 func TestMain(m *testing.M) {
 	teardown := setup()
 	code := m.Run()
@@ -21,10 +33,10 @@ func TestMain(m *testing.M) {
 func setup() func() {
 	if terraformBin != "" {
 		// this is pre-set when we're running in a binary produced from
-		// the make-archive.sh script, since that builds a ready-to-go
+		// the make-archive.sh script, since that is for testing an
-		// binary into the archive. However, we do need to turn it into
+		// executable obtained from a real release package. However, we do
-		// an absolute path so that we can find it when we change the
+		// need to turn it into an absolute path so that we can find it
-		// working directory during tests.
+		// when we change the working directory during tests.
 		var err error
 		terraformBin, err = filepath.Abs(terraformBin)
 		if err != nil {
@@ -38,6 +50,11 @@ func setup() func() {
 	// Make the executable available for use in tests
 	terraformBin = tmpFilename
 
+	// Tests running in the ad-hoc testing mode are allowed to use "go build"
+	// and similar to produce other test executables.
+	// (See the comment on this variable's declaration for more information.)
+	canRunGoBuild = true
+
 	return func() {
 		os.Remove(tmpFilename)
 	}

internal/command/e2etest/make-archive.sh

@@ -13,9 +13,12 @@
 # and then executed as follows:
 #    set TF_ACC=1
 #    ./e2etest.exe
-# Since the test archive includes both the test fixtures and the compiled
+#
-# terraform executable along with this test program, the result is
+# Because separated e2etest harnesses are intended for testing against "real"
-# self-contained and does not require a local Go compiler on the target system.
+# release executables, the generated archives don't include a copy of
+# the Terraform executable. Instead, the caller of the tests must retrieve
+# and extract a release package into the working directory before running
+# the e2etest executable, so that "e2etest" can find and execute it.
 
 set +euo pipefail
 
@@ -33,10 +36,6 @@ mkdir -p "$OUTDIR"
 # We need the test fixtures available when we run the tests.
 cp -r testdata "$OUTDIR/testdata"
 
-# Bundle a copy of our binary so the target system doesn't need the go
-# compiler installed.
-go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform
-
 # Build the test program
 go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/internal/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/internal/command/e2etest
 

internal/command/e2etest/primary_test.go

@@ -204,13 +204,13 @@ func TestPrimaryChdirOption(t *testing.T) {
 	}
 
 	gotOutput := state.RootModule().OutputValues["cwd"]
-	wantOutputValue := cty.StringVal(tf.Path()) // path.cwd returns the original path, because path.root is how we get the overridden path
+	wantOutputValue := cty.StringVal(filepath.ToSlash(tf.Path())) // path.cwd returns the original path, because path.root is how we get the overridden path
 	if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
 		t.Errorf("incorrect value for cwd output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
 	}
 
 	gotOutput = state.RootModule().OutputValues["root"]
-	wantOutputValue = cty.StringVal(tf.Path("subdir")) // path.root is a relative path, but the text fixture uses abspath on it.
+	wantOutputValue = cty.StringVal(filepath.ToSlash(tf.Path("subdir"))) // path.root is a relative path, but the text fixture uses abspath on it.
 	if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
 		t.Errorf("incorrect value for root output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
 	}

internal/command/e2etest/provider_dev_test.go

@@ -18,6 +18,14 @@ import (
 // we normally do, so they can just overwrite the same local executable
 // in-place to iterate faster.
 func TestProviderDevOverrides(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
 	t.Parallel()
 
 	tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override")

internal/command/e2etest/provider_plugin_test.go

@@ -13,6 +13,14 @@ import (
 // TestProviderProtocols verifies that Terraform can execute provider plugins
 // with both supported protocol versions.
 func TestProviderProtocols(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
 	t.Parallel()
 
 	tf := e2e.NewBinary(terraformBin, "testdata/provider-plugin")

internal/command/e2etest/providers_tamper_test.go

@@ -41,12 +41,16 @@ func TestProviderTampering(t *testing.T) {
 
 	seedDir := tf.WorkDir()
 	const providerVersion = "3.1.0" // must match the version in the fixture config
-	pluginDir := ".terraform/providers/registry.terraform.io/hashicorp/null/" + providerVersion + "/" + getproviders.CurrentPlatform.String()
+	pluginDir := filepath.Join(".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String())
-	pluginExe := pluginDir + "/terraform-provider-null_v" + providerVersion + "_x5"
+	pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5")
 	if getproviders.CurrentPlatform.OS == "windows" {
 		pluginExe += ".exe" // ugh
 	}
 
+	// filepath.Join here to make sure we get the right path separator
+	// for whatever OS we're running these tests on.
+	providerCacheDir := filepath.Join(".terraform", "providers")
+
 	t.Run("cache dir totally gone", func(t *testing.T) {
 		tf := e2e.NewBinary(terraformBin, seedDir)
 		defer tf.Close()
@@ -61,7 +65,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 		if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -128,7 +132,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 		if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -237,7 +241,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 	})
@@ -260,7 +264,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 	})

internal/command/e2etest/provisioner_plugin_test.go

@@ -12,6 +12,14 @@ import (
 // TestProvisionerPlugin is a test that terraform can execute a 3rd party
 // provisioner plugin.
 func TestProvisionerPlugin(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provisioner executable")
+	}
 	t.Parallel()
 
 	// This test reaches out to releases.hashicorp.com to download the