helper/variables: helpers for working with vars

helper/variables/flag.go

@@ -0,0 +1,151 @@
+package variables
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/hcl"
+)
+
+// Flag a flag.Value implementation for parsing user variables
+// from the command-line in the format of '-var key=value', where value is
+// a type intended for use as a Terraform variable.
+type Flag map[string]interface{}
+
+func (v *Flag) String() string {
+	return ""
+}
+
+func (v *Flag) Set(raw string) error {
+	key, value, err := parseVarFlag(raw)
+	if err != nil {
+		return err
+	}
+
+	*v = Merge(*v, map[string]interface{}{key: value})
+	return nil
+}
+
+var (
+	// This regular expression is how we check if a value for a variable
+	// matches what we'd expect a rich HCL value to be. For example: {
+	// definitely signals a map. If a value DOESN'T match this, we return
+	// it as a raw string.
+	varFlagHCLRe = regexp.MustCompile(`^["\[\{]`)
+)
+
+// parseVarFlag parses the value of a single variable specified
+// on the command line via -var or in an environment variable named TF_VAR_x,
+// where x is  the name of the variable. In order to get around the restriction
+// of HCL requiring a top level object, we prepend a sentinel key, decode the
+// user-specified value as its value and pull the value back out of the
+// resulting map.
+func parseVarFlag(input string) (string, interface{}, error) {
+	idx := strings.Index(input, "=")
+	if idx == -1 {
+		return "", nil, fmt.Errorf("No '=' value in variable: %s", input)
+	}
+
+	probablyName := input[0:idx]
+	value := input[idx+1:]
+	trimmed := strings.TrimSpace(value)
+
+	// If the value is a simple number, don't parse it as hcl because the
+	// variable type may actually be a string, and HCL will convert it to the
+	// numberic value. We could check this in the validation later, but the
+	// conversion may alter the string value.
+	if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
+		return probablyName, value, nil
+	}
+	if _, err := strconv.ParseFloat(trimmed, 64); err == nil {
+		return probablyName, value, nil
+	}
+
+	// HCL will also parse hex as a number
+	if strings.HasPrefix(trimmed, "0x") {
+		if _, err := strconv.ParseInt(trimmed[2:], 16, 64); err == nil {
+			return probablyName, value, nil
+		}
+	}
+
+	// If the value is a boolean value, also convert it to a simple string
+	// since Terraform core doesn't accept primitives as anything other
+	// than string for now.
+	if _, err := strconv.ParseBool(trimmed); err == nil {
+		return probablyName, value, nil
+	}
+
+	parsed, err := hcl.Parse(input)
+	if err != nil {
+		// If it didn't parse as HCL, we check if it doesn't match our
+		// whitelist of TF-accepted HCL types for inputs. If not, then
+		// we let it through as a raw string.
+		if !varFlagHCLRe.MatchString(trimmed) {
+			return probablyName, value, nil
+		}
+
+		// This covers flags of the form `foo=bar` which is not valid HCL
+		// At this point, probablyName is actually the name, and the remainder
+		// of the expression after the equals sign is the value.
+		if regexp.MustCompile(`Unknown token: \d+:\d+ IDENT`).Match([]byte(err.Error())) {
+			return probablyName, value, nil
+		}
+
+		return "", nil, fmt.Errorf(
+			"Cannot parse value for variable %s (%q) as valid HCL: %s",
+			probablyName, input, err)
+	}
+
+	var decoded map[string]interface{}
+	if hcl.DecodeObject(&decoded, parsed); err != nil {
+		return "", nil, fmt.Errorf(
+			"Cannot parse value for variable %s (%q) as valid HCL: %s",
+			probablyName, input, err)
+	}
+
+	// Cover cases such as key=
+	if len(decoded) == 0 {
+		return probablyName, "", nil
+	}
+
+	if len(decoded) > 1 {
+		return "", nil, fmt.Errorf(
+			"Cannot parse value for variable %s (%q) as valid HCL. "+
+				"Only one value may be specified.",
+			probablyName, input)
+	}
+
+	err = flattenMultiMaps(decoded)
+	if err != nil {
+		return probablyName, "", err
+	}
+
+	var k string
+	var v interface{}
+	for k, v = range decoded {
+		break
+	}
+
+	return k, v, nil
+}
+
+// Variables don't support any type that can be configured via multiple
+// declarations of the same HCL map, so any instances of
+// []map[string]interface{} are either a single map that can be flattened, or
+// are invalid config.
+func flattenMultiMaps(m map[string]interface{}) error {
+	for k, v := range m {
+		switch v := v.(type) {
+		case []map[string]interface{}:
+			switch {
+			case len(v) > 1:
+				return fmt.Errorf("multiple map declarations not supported for variables")
+			case len(v) == 1:
+				m[k] = v[0]
+			}
+		}
+	}
+	return nil
+}

helper/variables/flag_file.go

@@ -0,0 +1,65 @@
+package variables
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/hashicorp/hcl"
+	"github.com/mitchellh/go-homedir"
+)
+
+// FlagFile is a flag.Value implementation for parsing user variables
+// from the command line in the form of files. i.e. '-var-file=foo'
+type FlagFile map[string]interface{}
+
+func (v *FlagFile) String() string {
+	return ""
+}
+
+func (v *FlagFile) Set(raw string) error {
+	vs, err := loadKVFile(raw)
+	if err != nil {
+		return err
+	}
+
+	*v = Merge(*v, vs)
+	return nil
+}
+
+func loadKVFile(rawPath string) (map[string]interface{}, error) {
+	path, err := homedir.Expand(rawPath)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"Error expanding path: %s", err)
+	}
+
+	// Read the HCL file and prepare for parsing
+	d, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"Error reading %s: %s", path, err)
+	}
+
+	// Parse it
+	obj, err := hcl.Parse(string(d))
+	if err != nil {
+		return nil, fmt.Errorf(
+			"Error parsing %s: %s", path, err)
+	}
+
+	var result map[string]interface{}
+	if err := hcl.DecodeObject(&result, obj); err != nil {
+		return nil, fmt.Errorf(
+			"Error decoding Terraform vars file: %s\n\n"+
+				"The vars file should be in the format of `key = \"value\"`.\n"+
+				"Decoding errors are usually caused by an invalid format.",
+			err)
+	}
+
+	err = flattenMultiMaps(result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}

helper/variables/flag_file_test.go

@@ -0,0 +1,107 @@
+package variables
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func TestFlagFile_impl(t *testing.T) {
+	var _ flag.Value = new(FlagFile)
+}
+
+func TestFlagFile(t *testing.T) {
+	inputLibucl := `
+foo = "bar"
+`
+	inputMap := `
+foo = {
+	k = "v"
+}`
+
+	inputJson := `{
+		"foo": "bar"}`
+
+	cases := []struct {
+		Input  interface{}
+		Output map[string]interface{}
+		Error  bool
+	}{
+		{
+			inputLibucl,
+			map[string]interface{}{"foo": "bar"},
+			false,
+		},
+
+		{
+			inputJson,
+			map[string]interface{}{"foo": "bar"},
+			false,
+		},
+
+		{
+			`map.key = "foo"`,
+			map[string]interface{}{"map.key": "foo"},
+			false,
+		},
+
+		{
+			inputMap,
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"k": "v",
+				},
+			},
+			false,
+		},
+
+		{
+			[]string{
+				`foo = { "k" = "v"}`,
+				`foo = { "j" = "v" }`,
+			},
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"k": "v",
+					"j": "v",
+				},
+			},
+			false,
+		},
+	}
+
+	path := testTempFile(t)
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var input []string
+			switch i := tc.Input.(type) {
+			case string:
+				input = []string{i}
+			case []string:
+				input = i
+			default:
+				t.Fatalf("bad input type: %T", i)
+			}
+
+			f := new(FlagFile)
+			for _, input := range input {
+				if err := ioutil.WriteFile(path, []byte(input), 0644); err != nil {
+					t.Fatalf("err: %s", err)
+				}
+
+				err := f.Set(path)
+				if err != nil != tc.Error {
+					t.Fatalf("bad error. Input: %#v, err: %s", input, err)
+				}
+			}
+
+			actual := map[string]interface{}(*f)
+			if !reflect.DeepEqual(actual, tc.Output) {
+				t.Fatalf("bad: %#v", actual)
+			}
+		})
+	}
+}

helper/variables/flag_test.go

@@ -0,0 +1,174 @@
+package variables
+
+import (
+	"flag"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+func TestFlag_impl(t *testing.T) {
+	var _ flag.Value = new(Flag)
+}
+
+func TestFlag(t *testing.T) {
+	cases := []struct {
+		Input  interface{}
+		Output map[string]interface{}
+		Error  bool
+	}{
+		{
+			"key=value",
+			map[string]interface{}{"key": "value"},
+			false,
+		},
+
+		{
+			"key=",
+			map[string]interface{}{"key": ""},
+			false,
+		},
+
+		{
+			"key=foo=bar",
+			map[string]interface{}{"key": "foo=bar"},
+			false,
+		},
+
+		{
+			"key=false",
+			map[string]interface{}{"key": "false"},
+			false,
+		},
+
+		{
+			"map.key=foo",
+			map[string]interface{}{"map.key": "foo"},
+			false,
+		},
+
+		{
+			"key",
+			nil,
+			true,
+		},
+
+		{
+			`key=["hello", "world"]`,
+			map[string]interface{}{"key": []interface{}{"hello", "world"}},
+			false,
+		},
+
+		{
+			`key={"hello" = "world", "foo" = "bar"}`,
+			map[string]interface{}{
+				"key": map[string]interface{}{
+					"hello": "world",
+					"foo":   "bar",
+				},
+			},
+			false,
+		},
+
+		{
+			`key={"hello" = "world", "foo" = "bar"}\nkey2="invalid"`,
+			nil,
+			true,
+		},
+
+		{
+			"key=/path",
+			map[string]interface{}{"key": "/path"},
+			false,
+		},
+
+		{
+			"key=1234.dkr.ecr.us-east-1.amazonaws.com/proj:abcdef",
+			map[string]interface{}{"key": "1234.dkr.ecr.us-east-1.amazonaws.com/proj:abcdef"},
+			false,
+		},
+
+		// simple values that can parse as numbers should remain strings
+		{
+			"key=1",
+			map[string]interface{}{
+				"key": "1",
+			},
+			false,
+		},
+		{
+			"key=1.0",
+			map[string]interface{}{
+				"key": "1.0",
+			},
+			false,
+		},
+		{
+			"key=0x10",
+			map[string]interface{}{
+				"key": "0x10",
+			},
+			false,
+		},
+
+		// Test setting multiple times
+		{
+			[]string{
+				"foo=bar",
+				"bar=baz",
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": "baz",
+			},
+			false,
+		},
+
+		// Test map merging
+		{
+			[]string{
+				`foo={ foo = "bar" }`,
+				`foo={ bar = "baz" }`,
+			},
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"foo": "bar",
+					"bar": "baz",
+				},
+			},
+			false,
+		},
+	}
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("%d-%s", i, tc.Input), func(t *testing.T) {
+			var input []string
+			switch v := tc.Input.(type) {
+			case string:
+				input = []string{v}
+			case []string:
+				input = v
+			default:
+				t.Fatalf("bad input type: %T", tc.Input)
+			}
+
+			f := new(Flag)
+			for i, single := range input {
+				err := f.Set(single)
+
+				// Only check for expected errors on the final input
+				expected := tc.Error && i == len(input)-1
+				if err != nil != expected {
+					t.Fatalf("bad error. Input: %#v\n\nError: %s", single, err)
+				}
+			}
+
+			actual := map[string]interface{}(*f)
+			if !reflect.DeepEqual(actual, tc.Output) {
+				t.Fatalf("bad:\nexpected: %s\n\n     got: %s\n", spew.Sdump(tc.Output), spew.Sdump(actual))
+			}
+		})
+	}
+}

helper/variables/merge.go

@@ -0,0 +1,66 @@
+package variables
+
+// Merge merges raw variable values b into a.
+//
+// The parameters given here should be the full map of set variables, such
+// as those created by Flag and FlagFile.
+//
+// The merge behavior is to override the top-level key except for map
+// types. Map types are merged together by key. Any other types are overwritten:
+// primitives and lists.
+//
+// This returns the resulting map. This merges into a but if a is nil a new
+// map will be allocated. A non-nil "a" value is returned regardless.
+func Merge(a, b map[string]interface{}) map[string]interface{} {
+	if a == nil {
+		a = map[string]interface{}{}
+	}
+
+	for k, raw := range b {
+		switch v := raw.(type) {
+		case map[string]interface{}:
+			// For maps, we do a deep merge. If the value in the original
+			// map (a) is not a map, we just overwrite. For invalid types
+			// they're caught later in the validation step in Terraform.
+
+			// If there is no value set, just set it
+			rawA, ok := a[k]
+			if !ok {
+				a[k] = v
+				continue
+			}
+
+			// If the value is not a map, just set it
+			mapA, ok := rawA.(map[string]interface{})
+			if !ok {
+				a[k] = v
+				continue
+			}
+
+			// Go over the values in the map. If we're setting a raw value,
+			// then override. If we're setting a nested map, then recurse.
+			for k, v := range v {
+				// If the value isn't a map, then there is nothing to merge
+				// further so we just set it.
+				mv, ok := v.(map[string]interface{})
+				if !ok {
+					mapA[k] = v
+					continue
+				}
+
+				switch av := mapA[k].(type) {
+				case map[string]interface{}:
+					mapA[k] = Merge(av, mv)
+				default:
+					// Unset or non-map, just set it
+					mapA[k] = mv
+				}
+			}
+		default:
+			// Any other type we just set directly
+			a[k] = v
+		}
+	}
+
+	return a
+}

helper/variables/merge_test.go

@@ -0,0 +1,93 @@
+package variables
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestMerge(t *testing.T) {
+	cases := []struct {
+		Name     string
+		A, B     map[string]interface{}
+		Expected map[string]interface{}
+	}{
+		{
+			"basic key/value",
+			map[string]interface{}{
+				"foo": "bar",
+			},
+			map[string]interface{}{
+				"bar": "baz",
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": "baz",
+			},
+		},
+
+		{
+			"map unset",
+			map[string]interface{}{
+				"foo": "bar",
+			},
+			map[string]interface{}{
+				"bar": map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+		},
+
+		{
+			"map merge",
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": map[string]interface{}{
+					"bar": "baz",
+				},
+			},
+			map[string]interface{}{
+				"bar": map[string]interface{}{
+					"foo": "bar",
+				},
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": map[string]interface{}{
+					"foo": "bar",
+					"bar": "baz",
+				},
+			},
+		},
+
+		{
+			"basic k/v with lists",
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": []interface{}{"foo"},
+			},
+			map[string]interface{}{
+				"bar": []interface{}{"bar"},
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": []interface{}{"bar"},
+			},
+		},
+	}
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
+			actual := Merge(tc.A, tc.B)
+			if !reflect.DeepEqual(tc.Expected, actual) {
+				t.Fatalf("bad: %#v", actual)
+			}
+		})
+	}
+}

helper/variables/variables.go

@@ -0,0 +1,3 @@
+// Package variables provides functions and types for working with
+// Terraform variables provided as input.
+package variables

helper/variables/variables_test.go

@@ -0,0 +1,20 @@
+package variables
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+)
+
+func testTempFile(t *testing.T) string {
+	return filepath.Join(testTempDir(t), "temp.dat")
+}
+
+func testTempDir(t *testing.T) string {
+	d, err := ioutil.TempDir("", "tf")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return d
+}