From fee61a44b48df5fbce2491e4cee606d73f173356 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 7 Jul 2017 18:46:24 -0700 Subject: [PATCH] command/e2etest: end-to-end testing harness Previously we had no automated testing of whether we can produce a Terraform executable that actually works. Our various functional tests have good coverage of specific Terraform features and whole operations, but we lacked end-to-end testing of actual usage of the generated binary, without any stubbing. This package is intended as a vehicle for such end-to-end testing. When run normally under "go test" it will produce a build of the main Terraform binary and make it available for tests to execute. The harness exposes a flag for whether tests are allowed to reach out to external network services, controlled with our standard TF_ACC environment variable, so that basic local tests can be safely run as part of "make test" while more elaborate tests can be run easily when desired. It also provides a separate mode of operation where the included script make-archive.sh can be used to produce a self-contained test archive that can be copied to another system to run the tests there. This is intended to allow testing of cross-compiled binaries, by shipping them over to the target OS and architecture to run without requiring a full Go compiler installation on the target system. The goal here is not to test again functionality that's already well-covered by our existing tests, but rather to test chains of normal operations against the build binary that are not otherwise tested together. --- command/e2etest/.gitignore | 1 + command/e2etest/doc.go | 29 ++++ command/e2etest/main_test.go | 289 ++++++++++++++++++++++++++++++++ command/e2etest/make-archive.sh | 47 ++++++ 4 files changed, 366 insertions(+) create mode 100644 command/e2etest/.gitignore create mode 100644 command/e2etest/doc.go create mode 100644 command/e2etest/main_test.go create mode 100755 command/e2etest/make-archive.sh diff --git a/command/e2etest/.gitignore b/command/e2etest/.gitignore new file mode 100644 index 000000000..a007feab0 --- /dev/null +++ b/command/e2etest/.gitignore @@ -0,0 +1 @@ +build/* diff --git a/command/e2etest/doc.go b/command/e2etest/doc.go new file mode 100644 index 000000000..147cc48bc --- /dev/null +++ b/command/e2etest/doc.go @@ -0,0 +1,29 @@ +// Package e2etest contains a small number of tests that run against a real +// Terraform binary, compiled on the fly at the start of the test run. +// +// These tests help ensure that key end-to-end Terraform use-cases are working +// for a real binary, whereas other tests always have at least _some_ amount +// of test stubbing. +// +// The goal of this package is not to duplicate the functional testing done +// in other packages but rather to fully exercise a few important workflows +// in a realistic way. +// +// These tests can be used in two ways. The simplest way is to just run them +// with "go test" as normal: +// +// go test -v github.com/hashicorp/terraform/command/e2etest +// +// This will compile on the fly a Terraform binary and run the tests against +// it. +// +// Alternatively, the make-archive.sh script can be used to produce a +// self-contained zip file that can be shipped to another machine to run +// the tests there without needing a locally-installed Go compiler. This +// is primarily useful for testing cross-compiled builds. For more information, +// see the commentary in make-archive.sh. +// +// The TF_ACC environment variable must be set for the tests to reach out +// to external network services. Since these are end-to-end tests, only a +// few very basic tests can execute without this environment variable set. +package e2etest diff --git a/command/e2etest/main_test.go b/command/e2etest/main_test.go new file mode 100644 index 000000000..33a327976 --- /dev/null +++ b/command/e2etest/main_test.go @@ -0,0 +1,289 @@ +package e2etest + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + tfcore "github.com/hashicorp/terraform/terraform" +) + +var terraformBin string + +func TestMain(m *testing.M) { + teardown := setup() + code := m.Run() + teardown() + os.Exit(code) +} + +func setup() func() { + if terraformBin != "" { + // this is pre-set when we're running in a binary produced from + // the make-archive.sh script, since that builds a ready-to-go + // binary into the archive. However, we do need to turn it into + // an absolute path so that we can find it when we change the + // working directory during tests. + var err error + terraformBin, err = filepath.Abs(terraformBin) + if err != nil { + panic(fmt.Sprintf("failed to find absolute path of terraform executable: %s", err)) + } + return func() {} + } + + tmpFile, err := ioutil.TempFile("", "terraform") + if err != nil { + panic(err) + } + tmpFilename := tmpFile.Name() + if err = tmpFile.Close(); err != nil { + panic(err) + } + + cmd := exec.Command( + "go", "build", + "-o", tmpFilename, + "github.com/hashicorp/terraform", + ) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + err = cmd.Run() + if err != nil { + // The go compiler will have already produced some error messages + // on stderr by the time we get here. + panic(fmt.Sprintf("failed to build terraform executable: %s", err)) + } + + // Make the executable available for use in tests + terraformBin = tmpFilename + + return func() { + os.Remove(tmpFilename) + } +} + +func canAccessNetwork() bool { + // We re-use the flag normally used for acceptance tests since that's + // established as a way to opt-in to reaching out to real systems that + // may suffer transient errors. + return os.Getenv("TF_ACC") != "" +} + +func skipIfCannotAccessNetwork(t *testing.T) { + if !canAccessNetwork() { + t.Skip("network access not allowed; use TF_ACC=1 to enable") + } +} + +// Type terraform represents the combination of a compiled Terraform binary +// and a temporary working directory to run it in. +// +// This is the main harness for tests in this package. +type terraform struct { + bin string + dir string +} + +// newTerraform prepares a temporary directory containing the files from the +// given fixture and returns an instance of type terraform that can run +// the generated Terraform binary in that directory. +// +// If the temporary directory cannot be created, a fixture of the given name +// cannot be found, or if an error occurs while _copying_ the fixture files, +// this function will panic. Tests should be written to assume that this +// function always succeeds. +func newTerraform(fixtureName string) *terraform { + tmpDir, err := ioutil.TempDir("", "terraform-e2etest") + if err != nil { + panic(err) + } + + // For our purposes here we do a very simplistic file copy that doesn't + // attempt to preserve file permissions, attributes, alternate data + // streams, etc. Since we only have to deal with our own fixtures in + // the test-fixtures subdir, we know we don't need to deal with anything + // of this nature. + srcDir := filepath.Join("test-fixtures", fixtureName) + err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == srcDir { + // nothing to do at the root + return nil + } + + srcFn := path + + path, err = filepath.Rel(srcDir, path) + if err != nil { + return err + } + + dstFn := filepath.Join(tmpDir, path) + + if info.IsDir() { + return os.Mkdir(dstFn, os.ModePerm) + } + + src, err := os.Open(srcFn) + if err != nil { + return err + } + dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + + _, err = io.Copy(dst, src) + if err != nil { + return err + } + + if err := src.Close(); err != nil { + return err + } + if err := dst.Close(); err != nil { + return err + } + + return nil + }) + if err != nil { + panic(err) + } + + return &terraform{ + bin: terraformBin, + dir: tmpDir, + } +} + +// Cmd returns an exec.Cmd pre-configured to run the generated Terraform +// binary with the given arguments in the temporary working directory. +// +// The returned object can be mutated by the caller to customize how the +// process will be run, before calling Run. +func (t *terraform) Cmd(args ...string) *exec.Cmd { + cmd := exec.Command(t.bin, args...) + cmd.Dir = t.dir + cmd.Env = os.Environ() + + // Disable checkpoint since we don't want to harass that service when + // our tests run. (This does, of course, mean we can't actually do + // end-to-end testing of our Checkpoint interactions.) + cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1") + + return cmd +} + +// Run executes the generated Terraform binary with the given arguments +// and returns the bytes that it wrote to both stdout and stderr. +// +// This is a simple way to run Terraform for non-interactive commands +// that don't need any special environment variables. For more complex +// situations, use Cmd and customize the command before running it. +func (t *terraform) Run(args ...string) (stdout, stderr string, err error) { + cmd := t.Cmd(args...) + cmd.Stdin = nil + cmd.Stdout = &bytes.Buffer{} + cmd.Stderr = &bytes.Buffer{} + err = cmd.Run() + stdout = cmd.Stdout.(*bytes.Buffer).String() + stderr = cmd.Stderr.(*bytes.Buffer).String() + return +} + +// Path returns a file path within the temporary working directory by +// appending the given arguments as path segments. +func (t *terraform) Path(parts ...string) string { + args := make([]string, len(parts)+1) + args[0] = t.dir + args = append(args, parts...) + return filepath.Join(args...) +} + +// OpenFile is a helper for easily opening a file from the working directory +// for reading. +func (t *terraform) OpenFile(path ...string) (*os.File, error) { + flatPath := t.Path(path...) + return os.Open(flatPath) +} + +// ReadFile is a helper for easily reading a whole file from the working +// directory. +func (t *terraform) ReadFile(path ...string) ([]byte, error) { + flatPath := t.Path(path...) + return ioutil.ReadFile(flatPath) +} + +// FileExists is a helper for easily testing whether a particular file +// exists in the working directory. +func (t *terraform) FileExists(path ...string) bool { + flatPath := t.Path(path...) + _, err := os.Stat(flatPath) + return !os.IsNotExist(err) +} + +// LocalState is a helper for easily reading the local backend's state file +// terraform.tfstate from the working directory. +func (t *terraform) LocalState() (*tfcore.State, error) { + f, err := t.OpenFile("terraform.tfstate") + if err != nil { + return nil, err + } + defer f.Close() + return tfcore.ReadState(f) +} + +// Plan is a helper for easily reading a plan file from the working directory. +func (t *terraform) Plan(path ...string) (*tfcore.Plan, error) { + f, err := t.OpenFile(path...) + if err != nil { + return nil, err + } + defer f.Close() + return tfcore.ReadPlan(f) +} + +// SetLocalState is a helper for easily writing to the file the local backend +// uses for state in the working directory. This does not go through the +// actual local backend code, so processing such as management of serials +// does not apply and the given state will simply be written verbatim. +func (t *terraform) SetLocalState(state *tfcore.State) error { + path := t.Path("terraform.tfstate") + f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + defer func() { + err := f.Close() + if err != nil { + panic(fmt.Sprintf("failed to close state file after writing: %s", err)) + } + }() + + return tfcore.WriteState(state, f) +} + +// Close cleans up the temporary resources associated with the object, +// including its working directory. It is not valid to call Cmd or Run +// after Close returns. +// +// This method does _not_ stop any running child processes. It's the +// caller's responsibility to also terminate those _before_ closing the +// underlying terraform object. +// +// This function is designed to run under "defer", so it doesn't actually +// do any error handling and will leave dangling temporary files on disk +// if any errors occur while cleaning up. +func (t *terraform) Close() { + os.RemoveAll(t.dir) +} diff --git a/command/e2etest/make-archive.sh b/command/e2etest/make-archive.sh new file mode 100755 index 000000000..fa8e25ee6 --- /dev/null +++ b/command/e2etest/make-archive.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# For normal use this package can just be tested with "go test" as standard, +# but this script is an alternative to allow the tests to be run somewhere +# other than where they are built. + +# The primary use for this is cross-compilation, where e.g. we can produce an +# archive that can be extracted on a Windows system to run the e2e tests there: +# $ GOOS=windows GOARCH=amd64 ./make-archive.sh +# +# This will produce a zip file build/terraform-s2stest_windows_amd64.zip which +# can be shipped off to a Windows amd64 system, extracted to some directory, +# and then executed as follows: +# set TF_ACC=1 +# ./e2etest.exe +# Since the test archive includes both the test fixtures and the compiled +# terraform executable along with this test program, the result is +# self-contained and does not require a local Go compiler on the target system. + +set +euo pipefail + +# Always run from the directory where this script lives +cd "$( dirname "${BASH_SOURCE[0]}" )" + +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +GOEXE="$(go env GOEXE)" +OUTDIR="build/${GOOS}_${GOARCH}" +OUTFILE="terraform-e2etest_${GOOS}_${GOARCH}.zip" + +mkdir -p "$OUTDIR" + +# We need the test fixtures available when we run the tests. +cp -r test-fixtures "$OUTDIR/test-fixtures" + +# Bundle a copy of our binary so the target system doesn't need the go +# compiler installed. +go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform + +# Build the test program +go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/command/e2etest + +# Now bundle it all together for easy shipping! +cd "$OUTDIR" +zip -r "../$OUTFILE" * + +echo "e2etest archive created at build/$OUTFILE"