command/e2etest: end-to-end testing harness
Previously we had no automated testing of whether we can produce a Terraform executable that actually works. Our various functional tests have good coverage of specific Terraform features and whole operations, but we lacked end-to-end testing of actual usage of the generated binary, without any stubbing. This package is intended as a vehicle for such end-to-end testing. When run normally under "go test" it will produce a build of the main Terraform binary and make it available for tests to execute. The harness exposes a flag for whether tests are allowed to reach out to external network services, controlled with our standard TF_ACC environment variable, so that basic local tests can be safely run as part of "make test" while more elaborate tests can be run easily when desired. It also provides a separate mode of operation where the included script make-archive.sh can be used to produce a self-contained test archive that can be copied to another system to run the tests there. This is intended to allow testing of cross-compiled binaries, by shipping them over to the target OS and architecture to run without requiring a full Go compiler installation on the target system. The goal here is not to test again functionality that's already well-covered by our existing tests, but rather to test chains of normal operations against the build binary that are not otherwise tested together.
This commit is contained in:
Martin Atkins
parent
50f412bff4
commit
fee61a44b4
|
|
@ -0,0 +1 @@
|
|||
build/*
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
// Package e2etest contains a small number of tests that run against a real
|
||||
// Terraform binary, compiled on the fly at the start of the test run.
|
||||
//
|
||||
// These tests help ensure that key end-to-end Terraform use-cases are working
|
||||
// for a real binary, whereas other tests always have at least _some_ amount
|
||||
// of test stubbing.
|
||||
//
|
||||
// The goal of this package is not to duplicate the functional testing done
|
||||
// in other packages but rather to fully exercise a few important workflows
|
||||
// in a realistic way.
|
||||
//
|
||||
// These tests can be used in two ways. The simplest way is to just run them
|
||||
// with "go test" as normal:
|
||||
//
|
||||
// go test -v github.com/hashicorp/terraform/command/e2etest
|
||||
//
|
||||
// This will compile on the fly a Terraform binary and run the tests against
|
||||
// it.
|
||||
//
|
||||
// Alternatively, the make-archive.sh script can be used to produce a
|
||||
// self-contained zip file that can be shipped to another machine to run
|
||||
// the tests there without needing a locally-installed Go compiler. This
|
||||
// is primarily useful for testing cross-compiled builds. For more information,
|
||||
// see the commentary in make-archive.sh.
|
||||
//
|
||||
// The TF_ACC environment variable must be set for the tests to reach out
|
||||
// to external network services. Since these are end-to-end tests, only a
|
||||
// few very basic tests can execute without this environment variable set.
|
||||
package e2etest
|
||||
|
|
@ -0,0 +1,289 @@
|
|||
package e2etest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
tfcore "github.com/hashicorp/terraform/terraform"
|
||||
)
|
||||
|
||||
var terraformBin string
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
teardown := setup()
|
||||
code := m.Run()
|
||||
teardown()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func setup() func() {
|
||||
if terraformBin != "" {
|
||||
// this is pre-set when we're running in a binary produced from
|
||||
// the make-archive.sh script, since that builds a ready-to-go
|
||||
// binary into the archive. However, we do need to turn it into
|
||||
// an absolute path so that we can find it when we change the
|
||||
// working directory during tests.
|
||||
var err error
|
||||
terraformBin, err = filepath.Abs(terraformBin)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to find absolute path of terraform executable: %s", err))
|
||||
}
|
||||
return func() {}
|
||||
}
|
||||
|
||||
tmpFile, err := ioutil.TempFile("", "terraform")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
tmpFilename := tmpFile.Name()
|
||||
if err = tmpFile.Close(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cmd := exec.Command(
|
||||
"go", "build",
|
||||
"-o", tmpFilename,
|
||||
"github.com/hashicorp/terraform",
|
||||
)
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
// The go compiler will have already produced some error messages
|
||||
// on stderr by the time we get here.
|
||||
panic(fmt.Sprintf("failed to build terraform executable: %s", err))
|
||||
}
|
||||
|
||||
// Make the executable available for use in tests
|
||||
terraformBin = tmpFilename
|
||||
|
||||
return func() {
|
||||
os.Remove(tmpFilename)
|
||||
}
|
||||
}
|
||||
|
||||
func canAccessNetwork() bool {
|
||||
// We re-use the flag normally used for acceptance tests since that's
|
||||
// established as a way to opt-in to reaching out to real systems that
|
||||
// may suffer transient errors.
|
||||
return os.Getenv("TF_ACC") != ""
|
||||
}
|
||||
|
||||
func skipIfCannotAccessNetwork(t *testing.T) {
|
||||
if !canAccessNetwork() {
|
||||
t.Skip("network access not allowed; use TF_ACC=1 to enable")
|
||||
}
|
||||
}
|
||||
|
||||
// Type terraform represents the combination of a compiled Terraform binary
|
||||
// and a temporary working directory to run it in.
|
||||
//
|
||||
// This is the main harness for tests in this package.
|
||||
type terraform struct {
|
||||
bin string
|
||||
dir string
|
||||
}
|
||||
|
||||
// newTerraform prepares a temporary directory containing the files from the
|
||||
// given fixture and returns an instance of type terraform that can run
|
||||
// the generated Terraform binary in that directory.
|
||||
//
|
||||
// If the temporary directory cannot be created, a fixture of the given name
|
||||
// cannot be found, or if an error occurs while _copying_ the fixture files,
|
||||
// this function will panic. Tests should be written to assume that this
|
||||
// function always succeeds.
|
||||
func newTerraform(fixtureName string) *terraform {
|
||||
tmpDir, err := ioutil.TempDir("", "terraform-e2etest")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// For our purposes here we do a very simplistic file copy that doesn't
|
||||
// attempt to preserve file permissions, attributes, alternate data
|
||||
// streams, etc. Since we only have to deal with our own fixtures in
|
||||
// the test-fixtures subdir, we know we don't need to deal with anything
|
||||
// of this nature.
|
||||
srcDir := filepath.Join("test-fixtures", fixtureName)
|
||||
err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if path == srcDir {
|
||||
// nothing to do at the root
|
||||
return nil
|
||||
}
|
||||
|
||||
srcFn := path
|
||||
|
||||
path, err = filepath.Rel(srcDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dstFn := filepath.Join(tmpDir, path)
|
||||
|
||||
if info.IsDir() {
|
||||
return os.Mkdir(dstFn, os.ModePerm)
|
||||
}
|
||||
|
||||
src, err := os.Open(srcFn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(dst, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := src.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := dst.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return &terraform{
|
||||
bin: terraformBin,
|
||||
dir: tmpDir,
|
||||
}
|
||||
}
|
||||
|
||||
// Cmd returns an exec.Cmd pre-configured to run the generated Terraform
|
||||
// binary with the given arguments in the temporary working directory.
|
||||
//
|
||||
// The returned object can be mutated by the caller to customize how the
|
||||
// process will be run, before calling Run.
|
||||
func (t *terraform) Cmd(args ...string) *exec.Cmd {
|
||||
cmd := exec.Command(t.bin, args...)
|
||||
cmd.Dir = t.dir
|
||||
cmd.Env = os.Environ()
|
||||
|
||||
// Disable checkpoint since we don't want to harass that service when
|
||||
// our tests run. (This does, of course, mean we can't actually do
|
||||
// end-to-end testing of our Checkpoint interactions.)
|
||||
cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1")
|
||||
|
||||
return cmd
|
||||
}
|
||||
|
||||
// Run executes the generated Terraform binary with the given arguments
|
||||
// and returns the bytes that it wrote to both stdout and stderr.
|
||||
//
|
||||
// This is a simple way to run Terraform for non-interactive commands
|
||||
// that don't need any special environment variables. For more complex
|
||||
// situations, use Cmd and customize the command before running it.
|
||||
func (t *terraform) Run(args ...string) (stdout, stderr string, err error) {
|
||||
cmd := t.Cmd(args...)
|
||||
cmd.Stdin = nil
|
||||
cmd.Stdout = &bytes.Buffer{}
|
||||
cmd.Stderr = &bytes.Buffer{}
|
||||
err = cmd.Run()
|
||||
stdout = cmd.Stdout.(*bytes.Buffer).String()
|
||||
stderr = cmd.Stderr.(*bytes.Buffer).String()
|
||||
return
|
||||
}
|
||||
|
||||
// Path returns a file path within the temporary working directory by
|
||||
// appending the given arguments as path segments.
|
||||
func (t *terraform) Path(parts ...string) string {
|
||||
args := make([]string, len(parts)+1)
|
||||
args[0] = t.dir
|
||||
args = append(args, parts...)
|
||||
return filepath.Join(args...)
|
||||
}
|
||||
|
||||
// OpenFile is a helper for easily opening a file from the working directory
|
||||
// for reading.
|
||||
func (t *terraform) OpenFile(path ...string) (*os.File, error) {
|
||||
flatPath := t.Path(path...)
|
||||
return os.Open(flatPath)
|
||||
}
|
||||
|
||||
// ReadFile is a helper for easily reading a whole file from the working
|
||||
// directory.
|
||||
func (t *terraform) ReadFile(path ...string) ([]byte, error) {
|
||||
flatPath := t.Path(path...)
|
||||
return ioutil.ReadFile(flatPath)
|
||||
}
|
||||
|
||||
// FileExists is a helper for easily testing whether a particular file
|
||||
// exists in the working directory.
|
||||
func (t *terraform) FileExists(path ...string) bool {
|
||||
flatPath := t.Path(path...)
|
||||
_, err := os.Stat(flatPath)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
// LocalState is a helper for easily reading the local backend's state file
|
||||
// terraform.tfstate from the working directory.
|
||||
func (t *terraform) LocalState() (*tfcore.State, error) {
|
||||
f, err := t.OpenFile("terraform.tfstate")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return tfcore.ReadState(f)
|
||||
}
|
||||
|
||||
// Plan is a helper for easily reading a plan file from the working directory.
|
||||
func (t *terraform) Plan(path ...string) (*tfcore.Plan, error) {
|
||||
f, err := t.OpenFile(path...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return tfcore.ReadPlan(f)
|
||||
}
|
||||
|
||||
// SetLocalState is a helper for easily writing to the file the local backend
|
||||
// uses for state in the working directory. This does not go through the
|
||||
// actual local backend code, so processing such as management of serials
|
||||
// does not apply and the given state will simply be written verbatim.
|
||||
func (t *terraform) SetLocalState(state *tfcore.State) error {
|
||||
path := t.Path("terraform.tfstate")
|
||||
f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
err := f.Close()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to close state file after writing: %s", err))
|
||||
}
|
||||
}()
|
||||
|
||||
return tfcore.WriteState(state, f)
|
||||
}
|
||||
|
||||
// Close cleans up the temporary resources associated with the object,
|
||||
// including its working directory. It is not valid to call Cmd or Run
|
||||
// after Close returns.
|
||||
//
|
||||
// This method does _not_ stop any running child processes. It's the
|
||||
// caller's responsibility to also terminate those _before_ closing the
|
||||
// underlying terraform object.
|
||||
//
|
||||
// This function is designed to run under "defer", so it doesn't actually
|
||||
// do any error handling and will leave dangling temporary files on disk
|
||||
// if any errors occur while cleaning up.
|
||||
func (t *terraform) Close() {
|
||||
os.RemoveAll(t.dir)
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
#!/bin/bash
|
||||
|
||||
# For normal use this package can just be tested with "go test" as standard,
|
||||
# but this script is an alternative to allow the tests to be run somewhere
|
||||
# other than where they are built.
|
||||
|
||||
# The primary use for this is cross-compilation, where e.g. we can produce an
|
||||
# archive that can be extracted on a Windows system to run the e2e tests there:
|
||||
# $ GOOS=windows GOARCH=amd64 ./make-archive.sh
|
||||
#
|
||||
# This will produce a zip file build/terraform-s2stest_windows_amd64.zip which
|
||||
# can be shipped off to a Windows amd64 system, extracted to some directory,
|
||||
# and then executed as follows:
|
||||
# set TF_ACC=1
|
||||
# ./e2etest.exe
|
||||
# Since the test archive includes both the test fixtures and the compiled
|
||||
# terraform executable along with this test program, the result is
|
||||
# self-contained and does not require a local Go compiler on the target system.
|
||||
|
||||
set +euo pipefail
|
||||
|
||||
# Always run from the directory where this script lives
|
||||
cd "$( dirname "${BASH_SOURCE[0]}" )"
|
||||
|
||||
GOOS="$(go env GOOS)"
|
||||
GOARCH="$(go env GOARCH)"
|
||||
GOEXE="$(go env GOEXE)"
|
||||
OUTDIR="build/${GOOS}_${GOARCH}"
|
||||
OUTFILE="terraform-e2etest_${GOOS}_${GOARCH}.zip"
|
||||
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
# We need the test fixtures available when we run the tests.
|
||||
cp -r test-fixtures "$OUTDIR/test-fixtures"
|
||||
|
||||
# Bundle a copy of our binary so the target system doesn't need the go
|
||||
# compiler installed.
|
||||
go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform
|
||||
|
||||
# Build the test program
|
||||
go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/command/e2etest
|
||||
|
||||
# Now bundle it all together for easy shipping!
|
||||
cd "$OUTDIR"
|
||||
zip -r "../$OUTFILE" *
|
||||
|
||||
echo "e2etest archive created at build/$OUTFILE"
|
||||
Loading…
Reference in New Issue