Merge pull request #7803 from hashicorp/jbardin/tf_vars-push

Add tf_vars to the variables sent in push

command/hcl_printer.go

@@ -0,0 +1,190 @@
+package command
+
+// Marshal an object as an hcl value.
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+
+	"github.com/hashicorp/hcl/hcl/printer"
+)
+
+// This will only work operate on []interface{}, map[string]interface{}, and
+// primitive types.
+func encodeHCL(i interface{}) ([]byte, error) {
+	state := &encodeState{}
+	err := state.encode(i)
+	if err != nil {
+		return nil, err
+	}
+
+	hcl := state.Bytes()
+	if len(hcl) == 0 {
+		return hcl, nil
+	}
+
+	// the HCL parser requires an assignment. Strip it off again later
+	fakeAssignment := append([]byte("X = "), hcl...)
+
+	// use the real hcl parser to verify our output, and format it canonically
+	hcl, err = printer.Format(fakeAssignment)
+	if err != nil {
+		return nil, err
+	}
+
+	// now strip that first assignment off
+	eq := regexp.MustCompile(`=\s+`).FindIndex(hcl)
+
+	return hcl[eq[1]:], nil
+}
+
+type encodeState struct {
+	bytes.Buffer
+}
+
+func (e *encodeState) encode(i interface{}) error {
+	switch v := i.(type) {
+	case []interface{}:
+		return e.encodeList(v)
+
+	case map[string]interface{}:
+		return e.encodeMap(v)
+
+	case int, int8, int32, int64, uint8, uint32, uint64:
+		return e.encodeInt(i)
+
+	case float32, float64:
+		return e.encodeFloat(i)
+
+	case string:
+		return e.encodeString(v)
+
+	case nil:
+		return nil
+
+	default:
+		return fmt.Errorf("invalid type %T", i)
+	}
+
+}
+
+func (e *encodeState) encodeList(l []interface{}) error {
+	e.WriteString("[")
+	for i, v := range l {
+		err := e.encode(v)
+		if err != nil {
+			return err
+		}
+		if i < len(l)-1 {
+			e.WriteString(", ")
+		}
+	}
+	e.WriteString("]")
+	return nil
+}
+
+func (e *encodeState) encodeMap(m map[string]interface{}) error {
+	e.WriteString("{\n")
+	for i, k := range sortedKeys(m) {
+		v := m[k]
+
+		e.WriteString(k + " = ")
+		err := e.encode(v)
+		if err != nil {
+			return err
+		}
+		if i < len(m)-1 {
+			e.WriteString("\n")
+		}
+	}
+	e.WriteString("}")
+	return nil
+}
+
+func (e *encodeState) encodeInt(i interface{}) error {
+	_, err := fmt.Fprintf(e, "%d", i)
+	return err
+}
+
+func (e *encodeState) encodeFloat(f interface{}) error {
+	_, err := fmt.Fprintf(e, "%f", f)
+	return err
+}
+
+func (e *encodeState) encodeString(s string) error {
+	e.Write(quoteHCLString(s))
+	return nil
+}
+
+// Quote an HCL string, which may contain interpolations.
+// Since the string was already parsed from HCL, we have to assume the
+// required characters are sanely escaped. All we need to do is escape double
+// quotes in the string, unless they are in an interpolation block.
+func quoteHCLString(s string) []byte {
+	out := make([]byte, 0, len(s))
+	out = append(out, '"')
+
+	// our parse states
+	var (
+		outer  = 1 // the starting state for the string
+		dollar = 2 // look for '{' in the next character
+		interp = 3 // inside an interpolation block
+		escape = 4 // take the next character and pop back to prev state
+	)
+
+	// we could have nested interpolations
+	state := stack{}
+	state.push(outer)
+
+	for i := 0; i < len(s); i++ {
+		switch state.peek() {
+		case outer:
+			switch s[i] {
+			case '"':
+				out = append(out, '\\')
+			case '$':
+				state.push(dollar)
+			case '\\':
+				state.push(escape)
+			}
+		case dollar:
+			state.pop()
+			switch s[i] {
+			case '{':
+				state.push(interp)
+			case '\\':
+				state.push(escape)
+			}
+		case interp:
+			switch s[i] {
+			case '}':
+				state.pop()
+			}
+		case escape:
+			state.pop()
+		}
+
+		out = append(out, s[i])
+	}
+
+	out = append(out, '"')
+
+	return out
+}
+
+type stack []int
+
+func (s *stack) push(i int) {
+	*s = append(*s, i)
+}
+
+func (s *stack) pop() int {
+	last := len(*s) - 1
+	i := (*s)[last]
+	*s = (*s)[:last]
+	return i
+}
+
+func (s *stack) peek() int {
+	return (*s)[len(*s)-1]
+}

command/push.go

@@ -88,6 +88,7 @@ func (c *PushCommand) Run(args []string) int {
 		Path:      configPath,
 		StatePath: c.Meta.statePath,
 	})
+
 	if err != nil {
 		c.Ui.Error(err.Error())
 		return 1
@@ -136,19 +137,28 @@ func (c *PushCommand) Run(args []string) int {
 		c.client = &atlasPushClient{Client: client}
 	}
 
-	// Get the variables we might already have
+	// Get the variables we already have in atlas
 	atlasVars, err := c.client.Get(name)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf(
 			"Error looking up previously pushed configuration: %s", err))
 		return 1
 	}
-	for k, v := range atlasVars {
+
-		if _, ok := overwriteMap[k]; ok {
+	// filter any overwrites from the atlas vars
-			continue
+	for k := range overwriteMap {
+		delete(atlasVars, k)
 	}
 
-		ctx.SetVariable(k, v)
+	// Set remote variables in the context if we don't have a value here. These
+	// don't have to be correct, it just prevents the Input walk from prompting
+	// the user for input, The atlas variable may be an hcl-encoded object, but
+	// we're just going to set it as the raw string value.
+	ctxVars := ctx.Variables()
+	for k, av := range atlasVars {
+		if _, ok := ctxVars[k]; !ok {
+			ctx.SetVariable(k, av.Value)
+		}
 	}
 
 	// Ask for input
@@ -158,6 +168,18 @@ func (c *PushCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Now that we've gone through the input walk, we can be sure we have all
+	// the variables we're going to get.
+	// We are going to keep these separate from the atlas variables until
+	// upload, so we can notify the user which local variables we're sending.
+	serializedVars, err := tfVars(ctx.Variables())
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf(
+			"An error has occurred while serializing the variables for uploading:\n"+
+				"%s", err))
+		return 1
+	}
+
 	// Build the archiving options, which includes everything it can
 	// by default according to VCS rules but forcing the data directory.
 	archiveOpts := &archive.ArchiveOpts{
@@ -183,17 +205,23 @@ func (c *PushCommand) Run(args []string) int {
 
 	// Output to the user the variables that will be uploaded
 	var setVars []string
-	for k, _ := range ctx.Variables() {
+	// variables to upload
-		if _, ok := overwriteMap[k]; !ok {
+	var uploadVars []atlas.TFVar
-			if _, ok := atlasVars[k]; ok {
+
-				// Atlas variable not within override, so it came from Atlas
+	// Now we can combine the vars for upload to atlas and list the variables
-				continue
+	// we're uploading for the user
-			}
+	for _, sv := range serializedVars {
+		if av, ok := atlasVars[sv.Key]; ok {
+			// this belongs to Atlas
+			uploadVars = append(uploadVars, av)
+		} else {
+			// we're uploading our local version
+			setVars = append(setVars, sv.Key)
+			uploadVars = append(uploadVars, sv)
 		}
 
-		// This variable was set from the local value
-		setVars = append(setVars, k)
 	}
+
 	sort.Strings(setVars)
 	if len(setVars) > 0 {
 		c.Ui.Output(
@@ -214,7 +242,9 @@ func (c *PushCommand) Run(args []string) int {
 		Name:      name,
 		Archive:   archiveR,
 		Variables: ctx.Variables(),
+		TFVars:    uploadVars,
 	}
+
 	c.Ui.Output("Uploading Terraform configuration...")
 	vsn, err := c.client.Upsert(opts)
 	if err != nil {
@@ -272,14 +302,67 @@ Options:
 	return strings.TrimSpace(helpText)
 }
 
+func sortedKeys(m map[string]interface{}) []string {
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// build the set of TFVars for push
+func tfVars(vars map[string]interface{}) ([]atlas.TFVar, error) {
+	var tfVars []atlas.TFVar
+	var err error
+
+RANGE:
+	for _, k := range sortedKeys(vars) {
+		v := vars[k]
+
+		var hcl []byte
+		tfv := atlas.TFVar{Key: k}
+
+		switch v := v.(type) {
+		case string:
+			tfv.Value = v
+
+		case []interface{}:
+			hcl, err = encodeHCL(v)
+			if err != nil {
+				break RANGE
+			}
+
+			tfv.Value = string(hcl)
+			tfv.IsHCL = true
+
+		case map[string]interface{}:
+			hcl, err = encodeHCL(v)
+			if err != nil {
+				break RANGE
+			}
+
+			tfv.Value = string(hcl)
+			tfv.IsHCL = true
+		default:
+			err = fmt.Errorf("unknown type %T for variable %s", v, k)
+		}
+
+		tfVars = append(tfVars, tfv)
+	}
+
+	return tfVars, err
+}
+
 func (c *PushCommand) Synopsis() string {
 	return "Upload this Terraform module to Atlas to run"
 }
 
-// pushClient is implementd internally to control where pushes go. This is
+// pushClient is implemented internally to control where pushes go. This is
-// either to Atlas or a mock for testing.
+// either to Atlas or a mock for testing. We still return a map to make it
+// easier to check for variable existence when filtering the overrides.
 type pushClient interface {
-	Get(string) (map[string]interface{}, error)
+	Get(string) (map[string]atlas.TFVar, error)
 	Upsert(*pushUpsertOptions) (int, error)
 }
 
@@ -287,13 +370,14 @@ type pushUpsertOptions struct {
 	Name      string
 	Archive   *archive.Archive
 	Variables map[string]interface{}
+	TFVars    []atlas.TFVar
 }
 
 type atlasPushClient struct {
 	Client *atlas.Client
 }
 
-func (c *atlasPushClient) Get(name string) (map[string]interface{}, error) {
+func (c *atlasPushClient) Get(name string) (map[string]atlas.TFVar, error) {
 	user, name, err := atlas.ParseSlug(name)
 	if err != nil {
 		return nil, err
@@ -304,9 +388,21 @@ func (c *atlasPushClient) Get(name string) (map[string]interface{}, error) {
 		return nil, err
 	}
 
-	var variables map[string]interface{}
+	variables := make(map[string]atlas.TFVar)
-	if version != nil {
+
-		//variables = version.Variables
+	if version == nil {
+		return variables, nil
+	}
+
+	// Variables is superseded by TFVars
+	if version.TFVars == nil {
+		for k, v := range version.Variables {
+			variables[k] = atlas.TFVar{Key: k, Value: v}
+		}
+	} else {
+		for _, v := range version.TFVars {
+			variables[v.Key] = v
+		}
 	}
 
 	return variables, nil
@@ -319,7 +415,7 @@ func (c *atlasPushClient) Upsert(opts *pushUpsertOptions) (int, error) {
 	}
 
 	data := &atlas.TerraformConfigVersion{
-	//Variables: opts.Variables,
+		TFVars: opts.TFVars,
 	}
 
 	version, err := c.Client.CreateTerraformConfigVersion(
@@ -336,7 +432,7 @@ type mockPushClient struct {
 
 	GetCalled bool
 	GetName   string
-	GetResult map[string]interface{}
+	GetResult map[string]atlas.TFVar
 	GetError  error
 
 	UpsertCalled  bool
@@ -345,7 +441,7 @@ type mockPushClient struct {
 	UpsertError   error
 }
 
-func (c *mockPushClient) Get(name string) (map[string]interface{}, error) {
+func (c *mockPushClient) Get(name string) (map[string]atlas.TFVar, error) {
 	c.GetCalled = true
 	c.GetName = name
 	return c.GetResult, c.GetError

command/push_test.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"testing"
 
+	atlas "github.com/hashicorp/atlas-go/v1"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
 )
@@ -118,11 +119,13 @@ func TestPush_input(t *testing.T) {
 	variables := map[string]interface{}{
 		"foo": "foo",
 	}
+
 	if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) {
 		t.Fatalf("bad: %#v", client.UpsertOptions.Variables)
 	}
 }
 
+// We want a variable from atlas to fill a missing variable locally
 func TestPush_inputPartial(t *testing.T) {
 	tmp, cwd := testCwd(t)
 	defer testFixCwd(t, tmp, cwd)
@@ -143,7 +146,9 @@ func TestPush_inputPartial(t *testing.T) {
 
 	client := &mockPushClient{
 		File: archivePath,
-		GetResult: map[string]interface{}{"foo": "bar"},
+		GetResult: map[string]atlas.TFVar{
+			"foo": atlas.TFVar{Key: "foo", Value: "bar"},
+		},
 	}
 	ui := new(cli.MockUi)
 	c := &PushCommand{
@@ -170,12 +175,13 @@ func TestPush_inputPartial(t *testing.T) {
 		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
 
-	variables := map[string]interface{}{
+	expectedTFVars := []atlas.TFVar{
-		"foo": "bar",
+		{Key: "bar", Value: "foo"},
-		"bar": "foo",
+		{Key: "foo", Value: "bar"},
 	}
-	if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) {
+	if !reflect.DeepEqual(client.UpsertOptions.TFVars, expectedTFVars) {
-		t.Fatalf("bad: %#v", client.UpsertOptions)
+		t.Logf("expected: %#v", expectedTFVars)
+		t.Fatalf("got:      %#v", client.UpsertOptions.TFVars)
 	}
 }
 
@@ -208,8 +214,11 @@ func TestPush_localOverride(t *testing.T) {
 
 	client := &mockPushClient{File: archivePath}
 	// Provided vars should override existing ones
-	client.GetResult = map[string]interface{}{
+	client.GetResult = map[string]atlas.TFVar{
-		"foo": "old",
+		"foo": atlas.TFVar{
+			Key:   "foo",
+			Value: "old",
+		},
 	}
 	ui := new(cli.MockUi)
 	c := &PushCommand{
@@ -247,12 +256,11 @@ func TestPush_localOverride(t *testing.T) {
 		t.Fatalf("bad: %#v", client.UpsertOptions)
 	}
 
-	variables := map[string]interface{}{
+	expectedTFVars := pushTFVars()
-		"foo": "bar",
+
-		"bar": "foo",
+	if !reflect.DeepEqual(client.UpsertOptions.TFVars, expectedTFVars) {
-	}
+		t.Logf("expected: %#v", expectedTFVars)
-	if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) {
+		t.Fatalf("got:    %#v", client.UpsertOptions.TFVars)
-		t.Fatalf("bad: %#v", client.UpsertOptions)
 	}
 }
 
@@ -285,8 +293,11 @@ func TestPush_preferAtlas(t *testing.T) {
 
 	client := &mockPushClient{File: archivePath}
 	// Provided vars should override existing ones
-	client.GetResult = map[string]interface{}{
+	client.GetResult = map[string]atlas.TFVar{
-		"foo": "old",
+		"foo": atlas.TFVar{
+			Key:   "foo",
+			Value: "old",
+		},
 	}
 	ui := new(cli.MockUi)
 	c := &PushCommand{
@@ -323,12 +334,17 @@ func TestPush_preferAtlas(t *testing.T) {
 		t.Fatalf("bad: %#v", client.UpsertOptions)
 	}
 
-	variables := map[string]interface{}{
+	// change the expected response to match our change
-		"foo": "old",
+	expectedTFVars := pushTFVars()
-		"bar": "foo",
+	for i, v := range expectedTFVars {
+		if v.Key == "foo" {
+			expectedTFVars[i] = atlas.TFVar{Key: "foo", Value: "old"}
 		}
-	if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) {
+	}
-		t.Fatalf("bad: %#v", client.UpsertOptions)
+
+	if !reflect.DeepEqual(expectedTFVars, client.UpsertOptions.TFVars) {
+		t.Logf("expected: %#v", expectedTFVars)
+		t.Fatalf("got:      %#v", client.UpsertOptions.TFVars)
 	}
 }
 
@@ -394,12 +410,15 @@ func TestPush_tfvars(t *testing.T) {
 		t.Fatalf("bad: %#v", client.UpsertOptions)
 	}
 
-	variables := map[string]interface{}{
+	//now check TFVars
-		"foo": "bar",
+	tfvars := pushTFVars()
-		"bar": "foo",
+
+	for i, expected := range tfvars {
+		got := client.UpsertOptions.TFVars[i]
+		if got != expected {
+			t.Logf("%2d expected: %#v", i, expected)
+			t.Logf("        got: %#v", got)
 		}
-	if !reflect.DeepEqual(client.UpsertOptions.Variables, variables) {
-		t.Fatalf("bad: %#v", client.UpsertOptions)
 	}
 }
 
@@ -563,3 +582,25 @@ func testArchiveStr(t *testing.T, path string) []string {
 	sort.Strings(result)
 	return result
 }
+
+func pushTFVars() []atlas.TFVar {
+	return []atlas.TFVar{
+		{"bar", "foo", false},
+		{"baz", `{
+  A      = "a"
+  interp = "${file("t.txt")}"
+}
+`, true},
+		{"fob", `["a", "quotes \"in\" quotes"]` + "\n", true},
+		{"foo", "bar", false},
+	}
+}
+
+// the structure returned from the push-tfvars test fixture
+func pushTFVarsMap() map[string]atlas.TFVar {
+	vars := make(map[string]atlas.TFVar)
+	for _, v := range pushTFVars() {
+		vars[v.Key] = v
+	}
+	return vars
+}

command/test-fixtures/push-tfvars/main.tf

@@ -1,6 +1,21 @@
 variable "foo" {}
+
 variable "bar" {}
 
+variable "baz" {
+  type = "map"
+
+  default = {
+    "A"    = "a"
+    interp = "${file("t.txt")}"
+  }
+}
+
+variable "fob" {
+  type    = "list"
+  default = ["a", "quotes \"in\" quotes"]
+}
+
 resource "test_instance" "foo" {}
 
 atlas {

terraform/context.go

@@ -342,6 +342,7 @@ func (c *Context) Input(mode InputMode) error {
 
 			// Ask the user for a value for this variable
 			var value string
+			retry := 0
 			for {
 				var err error
 				value, err = c.uiInput.Input(&InputOpts{
@@ -355,7 +356,12 @@ func (c *Context) Input(mode InputMode) error {
 				}
 
 				if value == "" && v.Required() {
-					// Redo if it is required.
+					// Redo if it is required, but abort if we keep getting
+					// blank entries
+					if retry > 2 {
+						return fmt.Errorf("missing required value for %q", n)
+					}
+					retry++
 					continue
 				}
 

vendor/github.com/hashicorp/atlas-go/v1/terraform.go

@@ -14,7 +14,16 @@ type TerraformConfigVersion struct {
 	Version   int
 	Remotes   []string          `json:"remotes"`
 	Metadata  map[string]string `json:"metadata"`
-	Variables map[string]string `json:"variables"`
+	Variables map[string]string `json:"variables,omitempty"`
+	TFVars    []TFVar           `json:"tf_vars"`
+}
+
+// TFVar is used to serialize a single Terraform variable sent by the
+// manager as a collection of Variables in a Job payload.
+type TFVar struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+	IsHCL bool   `json:"hcl"`
 }
 
 // TerraformConfigLatest returns the latest Terraform configuration version.

vendor/vendor.json

@@ -1060,11 +1060,11 @@
 			"revision": "95fa852edca41c06c4ce526af4bb7dec4eaad434"
 		},
 		{
-			"checksumSHA1": "EWGfo74RcoKaYFZNSkvzYRJMgrY=",
+			"checksumSHA1": "yylO3hSRKd0T4mveT9ho2OSARwU=",
 			"comment": "20141209094003-92-g95fa852",
 			"path": "github.com/hashicorp/atlas-go/v1",
-			"revision": "c8b26aa95f096efc0f378b2d2830ca909631d584",
+			"revision": "9be9a611a15ba2f857a99b332fd966896867299a",
-			"revisionTime": "2016-07-22T13:58:36Z"
+			"revisionTime": "2016-07-26T16:33:11Z"
 		},
 		{
 			"comment": "v0.6.3-28-g3215b87",