Merge #15509: End-to-end test harness and some initial tests

command/e2etest/.gitignore

@@ -0,0 +1 @@
+build/*

command/e2etest/doc.go

@@ -0,0 +1,29 @@
+// Package e2etest contains a small number of tests that run against a real
+// Terraform binary, compiled on the fly at the start of the test run.
+//
+// These tests help ensure that key end-to-end Terraform use-cases are working
+// for a real binary, whereas other tests always have at least _some_ amount
+// of test stubbing.
+//
+// The goal of this package is not to duplicate the functional testing done
+// in other packages but rather to fully exercise a few important workflows
+// in a realistic way.
+//
+// These tests can be used in two ways. The simplest way is to just run them
+// with "go test" as normal:
+//
+//     go test -v github.com/hashicorp/terraform/command/e2etest
+//
+// This will compile on the fly a Terraform binary and run the tests against
+// it.
+//
+// Alternatively, the make-archive.sh script can be used to produce a
+// self-contained zip file that can be shipped to another machine to run
+// the tests there without needing a locally-installed Go compiler. This
+// is primarily useful for testing cross-compiled builds. For more information,
+// see the commentary in make-archive.sh.
+//
+// The TF_ACC environment variable must be set for the tests to reach out
+// to external network services. Since these are end-to-end tests, only a
+// few very basic tests can execute without this environment variable set.
+package e2etest

command/e2etest/init_test.go

@@ -0,0 +1,43 @@
+package e2etest
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestInitProviders(t *testing.T) {
+	t.Parallel()
+
+	// This test reaches out to releases.hashicorp.com to download the
+	// template provider, so it can only run if network access is allowed.
+	// We intentionally don't try to stub this here, because there's already
+	// a stubbed version of this in the "command" package and so the goal here
+	// is to test the interaction with the real repository.
+	skipIfCannotAccessNetwork(t)
+
+	tf := newTerraform("template-provider")
+	defer tf.Close()
+
+	stdout, stderr, err := tf.Run("init")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	if !strings.Contains(stdout, "Terraform has been successfully initialized!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+		t.Errorf("provider download message is missing from output:\n%s", stdout)
+		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
+	}
+
+	if !strings.Contains(stdout, "* provider.template: version = ") {
+		t.Errorf("provider pinning recommendation is missing from output:\n%s", stdout)
+	}
+
+}

command/e2etest/main_test.go

@@ -0,0 +1,289 @@
+package e2etest
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	tfcore "github.com/hashicorp/terraform/terraform"
+)
+
+var terraformBin string
+
+func TestMain(m *testing.M) {
+	teardown := setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}
+
+func setup() func() {
+	if terraformBin != "" {
+		// this is pre-set when we're running in a binary produced from
+		// the make-archive.sh script, since that builds a ready-to-go
+		// binary into the archive. However, we do need to turn it into
+		// an absolute path so that we can find it when we change the
+		// working directory during tests.
+		var err error
+		terraformBin, err = filepath.Abs(terraformBin)
+		if err != nil {
+			panic(fmt.Sprintf("failed to find absolute path of terraform executable: %s", err))
+		}
+		return func() {}
+	}
+
+	tmpFile, err := ioutil.TempFile("", "terraform")
+	if err != nil {
+		panic(err)
+	}
+	tmpFilename := tmpFile.Name()
+	if err = tmpFile.Close(); err != nil {
+		panic(err)
+	}
+
+	cmd := exec.Command(
+		"go", "build",
+		"-o", tmpFilename,
+		"github.com/hashicorp/terraform",
+	)
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+
+	err = cmd.Run()
+	if err != nil {
+		// The go compiler will have already produced some error messages
+		// on stderr by the time we get here.
+		panic(fmt.Sprintf("failed to build terraform executable: %s", err))
+	}
+
+	// Make the executable available for use in tests
+	terraformBin = tmpFilename
+
+	return func() {
+		os.Remove(tmpFilename)
+	}
+}
+
+func canAccessNetwork() bool {
+	// We re-use the flag normally used for acceptance tests since that's
+	// established as a way to opt-in to reaching out to real systems that
+	// may suffer transient errors.
+	return os.Getenv("TF_ACC") != ""
+}
+
+func skipIfCannotAccessNetwork(t *testing.T) {
+	if !canAccessNetwork() {
+		t.Skip("network access not allowed; use TF_ACC=1 to enable")
+	}
+}
+
+// Type terraform represents the combination of a compiled Terraform binary
+// and a temporary working directory to run it in.
+//
+// This is the main harness for tests in this package.
+type terraform struct {
+	bin string
+	dir string
+}
+
+// newTerraform prepares a temporary directory containing the files from the
+// given fixture and returns an instance of type terraform that can run
+// the generated Terraform binary in that directory.
+//
+// If the temporary directory cannot be created, a fixture of the given name
+// cannot be found, or if an error occurs while _copying_ the fixture files,
+// this function will panic. Tests should be written to assume that this
+// function always succeeds.
+func newTerraform(fixtureName string) *terraform {
+	tmpDir, err := ioutil.TempDir("", "terraform-e2etest")
+	if err != nil {
+		panic(err)
+	}
+
+	// For our purposes here we do a very simplistic file copy that doesn't
+	// attempt to preserve file permissions, attributes, alternate data
+	// streams, etc. Since we only have to deal with our own fixtures in
+	// the test-fixtures subdir, we know we don't need to deal with anything
+	// of this nature.
+	srcDir := filepath.Join("test-fixtures", fixtureName)
+	err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == srcDir {
+			// nothing to do at the root
+			return nil
+		}
+
+		srcFn := path
+
+		path, err = filepath.Rel(srcDir, path)
+		if err != nil {
+			return err
+		}
+
+		dstFn := filepath.Join(tmpDir, path)
+
+		if info.IsDir() {
+			return os.Mkdir(dstFn, os.ModePerm)
+		}
+
+		src, err := os.Open(srcFn)
+		if err != nil {
+			return err
+		}
+		dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(dst, src)
+		if err != nil {
+			return err
+		}
+
+		if err := src.Close(); err != nil {
+			return err
+		}
+		if err := dst.Close(); err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	return &terraform{
+		bin: terraformBin,
+		dir: tmpDir,
+	}
+}
+
+// Cmd returns an exec.Cmd pre-configured to run the generated Terraform
+// binary with the given arguments in the temporary working directory.
+//
+// The returned object can be mutated by the caller to customize how the
+// process will be run, before calling Run.
+func (t *terraform) Cmd(args ...string) *exec.Cmd {
+	cmd := exec.Command(t.bin, args...)
+	cmd.Dir = t.dir
+	cmd.Env = os.Environ()
+
+	// Disable checkpoint since we don't want to harass that service when
+	// our tests run. (This does, of course, mean we can't actually do
+	// end-to-end testing of our Checkpoint interactions.)
+	cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")
+
+	return cmd
+}
+
+// Run executes the generated Terraform binary with the given arguments
+// and returns the bytes that it wrote to both stdout and stderr.
+//
+// This is a simple way to run Terraform for non-interactive commands
+// that don't need any special environment variables. For more complex
+// situations, use Cmd and customize the command before running it.
+func (t *terraform) Run(args ...string) (stdout, stderr string, err error) {
+	cmd := t.Cmd(args...)
+	cmd.Stdin = nil
+	cmd.Stdout = &bytes.Buffer{}
+	cmd.Stderr = &bytes.Buffer{}
+	err = cmd.Run()
+	stdout = cmd.Stdout.(*bytes.Buffer).String()
+	stderr = cmd.Stderr.(*bytes.Buffer).String()
+	return
+}
+
+// Path returns a file path within the temporary working directory by
+// appending the given arguments as path segments.
+func (t *terraform) Path(parts ...string) string {
+	args := make([]string, len(parts)+1)
+	args[0] = t.dir
+	args = append(args, parts...)
+	return filepath.Join(args...)
+}
+
+// OpenFile is a helper for easily opening a file from the working directory
+// for reading.
+func (t *terraform) OpenFile(path ...string) (*os.File, error) {
+	flatPath := t.Path(path...)
+	return os.Open(flatPath)
+}
+
+// ReadFile is a helper for easily reading a whole file from the working
+// directory.
+func (t *terraform) ReadFile(path ...string) ([]byte, error) {
+	flatPath := t.Path(path...)
+	return ioutil.ReadFile(flatPath)
+}
+
+// FileExists is a helper for easily testing whether a particular file
+// exists in the working directory.
+func (t *terraform) FileExists(path ...string) bool {
+	flatPath := t.Path(path...)
+	_, err := os.Stat(flatPath)
+	return !os.IsNotExist(err)
+}
+
+// LocalState is a helper for easily reading the local backend's state file
+// terraform.tfstate from the working directory.
+func (t *terraform) LocalState() (*tfcore.State, error) {
+	f, err := t.OpenFile("terraform.tfstate")
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return tfcore.ReadState(f)
+}
+
+// Plan is a helper for easily reading a plan file from the working directory.
+func (t *terraform) Plan(path ...string) (*tfcore.Plan, error) {
+	f, err := t.OpenFile(path...)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return tfcore.ReadPlan(f)
+}
+
+// SetLocalState is a helper for easily writing to the file the local backend
+// uses for state in the working directory. This does not go through the
+// actual local backend code, so processing such as management of serials
+// does not apply and the given state will simply be written verbatim.
+func (t *terraform) SetLocalState(state *tfcore.State) error {
+	path := t.Path("terraform.tfstate")
+	f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			panic(fmt.Sprintf("failed to close state file after writing: %s", err))
+		}
+	}()
+
+	return tfcore.WriteState(state, f)
+}
+
+// Close cleans up the temporary resources associated with the object,
+// including its working directory. It is not valid to call Cmd or Run
+// after Close returns.
+//
+// This method does _not_ stop any running child processes. It's the
+// caller's responsibility to also terminate those _before_ closing the
+// underlying terraform object.
+//
+// This function is designed to run under "defer", so it doesn't actually
+// do any error handling and will leave dangling temporary files on disk
+// if any errors occur while cleaning up.
+func (t *terraform) Close() {
+	os.RemoveAll(t.dir)
+}

command/e2etest/make-archive.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# For normal use this package can just be tested with "go test" as standard,
+# but this script is an alternative to allow the tests to be run somewhere
+# other than where they are built.
+
+# The primary use for this is cross-compilation, where e.g. we can produce an
+# archive that can be extracted on a Windows system to run the e2e tests there:
+#    $ GOOS=windows GOARCH=amd64 ./make-archive.sh
+#
+# This will produce a zip file build/terraform-s2stest_windows_amd64.zip which
+# can be shipped off to a Windows amd64 system, extracted to some directory,
+# and then executed as follows:
+#    set TF_ACC=1
+#    ./e2etest.exe
+# Since the test archive includes both the test fixtures and the compiled
+# terraform executable along with this test program, the result is
+# self-contained and does not require a local Go compiler on the target system.
+
+set +euo pipefail
+
+# Always run from the directory where this script lives
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+
+GOOS="$(go env GOOS)"
+GOARCH="$(go env GOARCH)"
+GOEXE="$(go env GOEXE)"
+OUTDIR="build/${GOOS}_${GOARCH}"
+OUTFILE="terraform-e2etest_${GOOS}_${GOARCH}.zip"
+
+mkdir -p "$OUTDIR"
+
+# We need the test fixtures available when we run the tests.
+cp -r test-fixtures "$OUTDIR/test-fixtures"
+
+# Bundle a copy of our binary so the target system doesn't need the go
+# compiler installed.
+go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform
+
+# Build the test program
+go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/command/e2etest
+
+# Now bundle it all together for easy shipping!
+cd "$OUTDIR"
+zip -r "../$OUTFILE" *
+
+echo "e2etest archive created at build/$OUTFILE"

command/e2etest/primary_test.go

@@ -0,0 +1,123 @@
+package e2etest
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+// The tests in this file are for the "primary workflow", which includes
+// variants of the following sequence, with different details:
+// terraform init
+// terraform plan
+// terraform apply
+// terraform destroy
+
+func TestPrimarySeparatePlan(t *testing.T) {
+	t.Parallel()
+
+	// This test reaches out to releases.hashicorp.com to download the
+	// template and null providers, so it can only run if network access is
+	// allowed.
+	skipIfCannotAccessNetwork(t)
+
+	tf := newTerraform("full-workflow-null")
+	defer tf.Close()
+
+	//// INIT
+	stdout, stderr, err := tf.Run("init")
+	if err != nil {
+		t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	// Make sure we actually downloaded the plugins, rather than picking up
+	// copies that might be already installed globally on the system.
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") {
+		t.Errorf("template provider download message is missing from init output:\n%s", stdout)
+		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
+	}
+	if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") {
+		t.Errorf("null provider download message is missing from init output:\n%s", stdout)
+		t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)")
+	}
+
+	//// PLAN
+	stdout, stderr, err = tf.Run("plan", "-out=tfplan")
+	if err != nil {
+		t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "1 to add, 0 to change, 0 to destroy") {
+		t.Errorf("incorrect plan tally; want 1 to add:\n%s", stdout)
+	}
+
+	plan, err := tf.Plan("tfplan")
+	if err != nil {
+		t.Fatalf("failed to read plan file: %s", err)
+	}
+
+	stateResources := plan.State.RootModule().Resources
+	diffResources := plan.Diff.RootModule().Resources
+
+	if len(stateResources) != 1 || stateResources["data.template_file.test"] == nil {
+		t.Errorf("incorrect state in plan; want just data.template_file.test to have been rendered, but have:\n%s", spew.Sdump(stateResources))
+	}
+	if len(diffResources) != 1 || diffResources["null_resource.test"] == nil {
+		t.Errorf("incorrect diff in plan; want just null_resource.test to have been rendered, but have:\n%s", spew.Sdump(diffResources))
+	}
+
+	//// APPLY
+	stdout, stderr, err = tf.Run("apply", "tfplan")
+	if err != nil {
+		t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") {
+		t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout)
+	}
+
+	state, err := tf.LocalState()
+	if err != nil {
+		t.Fatalf("failed to read state file: %s", err)
+	}
+
+	stateResources = state.RootModule().Resources
+	var gotResources []string
+	for n := range stateResources {
+		gotResources = append(gotResources, n)
+	}
+	sort.Strings(gotResources)
+
+	wantResources := []string{
+		"data.template_file.test",
+		"null_resource.test",
+	}
+
+	if !reflect.DeepEqual(gotResources, wantResources) {
+		t.Errorf("wrong resources in state\ngot: %#v\nwant: %#v", gotResources, wantResources)
+	}
+
+	//// DESTROY
+	stdout, stderr, err = tf.Run("destroy", "-force")
+	if err != nil {
+		t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr)
+	}
+
+	if !strings.Contains(stdout, "Resources: 2 destroyed") {
+		t.Errorf("incorrect destroy tally; want 2 destroyed:\n%s", stdout)
+	}
+
+	state, err = tf.LocalState()
+	if err != nil {
+		t.Fatalf("failed to read state file after destroy: %s", err)
+	}
+
+	stateResources = state.RootModule().Resources
+	if len(stateResources) != 0 {
+		t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources))
+	}
+
+}

command/e2etest/test-fixtures/full-workflow-null/main.tf

@@ -0,0 +1,22 @@
+
+variable "name" {
+  default = "world"
+}
+
+data "template_file" "test" {
+  template = "Hello, $${name}"
+
+  vars = {
+    name = "${var.name}"
+  }
+}
+
+resource "null_resource" "test" {
+  triggers = {
+    greeting = "${data.template_file.test.rendered}"
+  }
+}
+
+output "greeting" {
+  value = "${null_resource.test.triggers["greeting"]}"
+}

command/e2etest/test-fixtures/template-provider/main.tf

@@ -0,0 +1,7 @@
+provider "template" {
+
+}
+
+data "template_file" "test" {
+  template = "Hello World"
+}

command/e2etest/version_test.go

@@ -0,0 +1,35 @@
+package e2etest
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	tfcore "github.com/hashicorp/terraform/terraform"
+)
+
+func TestVersion(t *testing.T) {
+	// Along with testing the "version" command in particular, this serves
+	// as a good smoke test for whether the Terraform binary can even be
+	// compiled and run, since it doesn't require any external network access
+	// to do its job.
+
+	t.Parallel()
+
+	tf := newTerraform("empty")
+	defer tf.Close()
+
+	stdout, stderr, err := tf.Run("version")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	wantVersion := fmt.Sprintf("Terraform %s", tfcore.VersionString())
+	if strings.Contains(stdout, wantVersion) {
+		t.Errorf("output does not contain our current version %q:\n%s", wantVersion, stdout)
+	}
+}