Merge pull request #12796 from hashicorp/b-init-kv

command/init: backend-config accepts key=value pairs

command/init.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/helper/variables"
 )
 
 // InitCommand is a Command implementation that takes a Terraform
@@ -19,11 +20,11 @@ type InitCommand struct {
 
 func (c *InitCommand) Run(args []string) int {
 	var flagBackend, flagGet bool
-	var flagConfigFile string
+	var flagConfigExtra map[string]interface{}
 	args = c.Meta.process(args, false)
 	cmdFlags := c.flagSet("init")
 	cmdFlags.BoolVar(&flagBackend, "backend", true, "")
-	cmdFlags.StringVar(&flagConfigFile, "backend-config", "", "")
+	cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "")
 	cmdFlags.BoolVar(&flagGet, "get", true, "")
 	cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
 	if err := cmdFlags.Parse(args); err != nil {
@@ -138,9 +139,9 @@ func (c *InitCommand) Run(args []string) int {
 			}
 
 			opts := &BackendOpts{
-				ConfigPath: path,
-				ConfigFile: flagConfigFile,
-				Init:       true,
+				ConfigPath:  path,
+				ConfigExtra: flagConfigExtra,
+				Init:        true,
 			}
 			if _, err := c.Backend(opts); err != nil {
 				c.Ui.Error(err.Error())
@@ -210,8 +211,12 @@ Options:
 
   -backend=true        Configure the backend for this environment.
 
-  -backend-config=path A path to load additional configuration for the backend.
-                       This is merged with what is in the configuration file.
+  -backend-config=path This can be either a path to an HCL file with key/value
+                       assignments (same format as terraform.tfvars) or a
+                       'key=value' format. This is merged with what is in the
+                       configuration file. This can be specified multiple
+                       times. The backend type must be in the configuration
+                       itself.
 
   -get=true            Download any modules for this configuration.
 

command/meta_backend.go

@@ -38,6 +38,10 @@ type BackendOpts struct {
 	// from a file.
 	ConfigFile string
 
+	// ConfigExtra is extra configuration to merge into the backend
+	// configuration after the extra file above.
+	ConfigExtra map[string]interface{}
+
 	// Plan is a plan that is being used. If this is set, the backend
 	// configuration and output configuration will come from this plan.
 	Plan *terraform.Plan
@@ -251,6 +255,20 @@ func (m *Meta) backendConfig(opts *BackendOpts) (*config.Backend, error) {
 		backend.RawConfig = backend.RawConfig.Merge(rc)
 	}
 
+	// If we have extra config values, merge that
+	if len(opts.ConfigExtra) > 0 {
+		log.Printf(
+			"[DEBUG] command: adding extra backend config from CLI")
+		rc, err := config.NewRawConfig(opts.ConfigExtra)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"Error adding extra configuration file for backend: %s", err)
+		}
+
+		// Merge in the configuration
+		backend.RawConfig = backend.RawConfig.Merge(rc)
+	}
+
 	// Validate the backend early. We have to do this before the normal
 	// config validation pass since backend loading happens earlier.
 	if errs := backend.Validate(); len(errs) > 0 {

helper/variables/flag_any.go

@@ -0,0 +1,25 @@
+package variables
+
+import (
+	"strings"
+)
+
+// FlagAny is a flag.Value for parsing user variables in the format of
+// 'key=value' OR a file path. 'key=value' is assumed if '=' is in the value.
+// You cannot use a file path that contains an '='.
+type FlagAny map[string]interface{}
+
+func (v *FlagAny) String() string {
+	return ""
+}
+
+func (v *FlagAny) Set(raw string) error {
+	idx := strings.Index(raw, "=")
+	if idx >= 0 {
+		flag := (*Flag)(v)
+		return flag.Set(raw)
+	}
+
+	flag := (*FlagFile)(v)
+	return flag.Set(raw)
+}

helper/variables/flag_any_test.go

@@ -0,0 +1,299 @@
+package variables
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+func TestFlagAny_impl(t *testing.T) {
+	var _ flag.Value = new(FlagAny)
+}
+
+func TestFlagAny(t *testing.T) {
+	cases := []struct {
+		Input  interface{}
+		Output map[string]interface{}
+		Error  bool
+	}{
+		{
+			"=value",
+			nil,
+			true,
+		},
+
+		{
+			" =value",
+			nil,
+			true,
+		},
+
+		{
+			"key=value",
+			map[string]interface{}{"key": "value"},
+			false,
+		},
+
+		{
+			"key=",
+			map[string]interface{}{"key": ""},
+			false,
+		},
+
+		{
+			"key=foo=bar",
+			map[string]interface{}{"key": "foo=bar"},
+			false,
+		},
+
+		{
+			"key=false",
+			map[string]interface{}{"key": "false"},
+			false,
+		},
+
+		{
+			"key =value",
+			map[string]interface{}{"key": "value"},
+			false,
+		},
+
+		{
+			"key = value",
+			map[string]interface{}{"key": " value"},
+			false,
+		},
+
+		{
+			`key = "value"`,
+			map[string]interface{}{"key": "value"},
+			false,
+		},
+
+		{
+			"map.key=foo",
+			map[string]interface{}{"map.key": "foo"},
+			false,
+		},
+
+		{
+			"key",
+			nil,
+			true,
+		},
+
+		{
+			`key=["hello", "world"]`,
+			map[string]interface{}{"key": []interface{}{"hello", "world"}},
+			false,
+		},
+
+		{
+			`key={"hello" = "world", "foo" = "bar"}`,
+			map[string]interface{}{
+				"key": map[string]interface{}{
+					"hello": "world",
+					"foo":   "bar",
+				},
+			},
+			false,
+		},
+
+		{
+			`key={"hello" = "world", "foo" = "bar"}\nkey2="invalid"`,
+			nil,
+			true,
+		},
+
+		{
+			"key=/path",
+			map[string]interface{}{"key": "/path"},
+			false,
+		},
+
+		{
+			"key=1234.dkr.ecr.us-east-1.amazonaws.com/proj:abcdef",
+			map[string]interface{}{"key": "1234.dkr.ecr.us-east-1.amazonaws.com/proj:abcdef"},
+			false,
+		},
+
+		// simple values that can parse as numbers should remain strings
+		{
+			"key=1",
+			map[string]interface{}{
+				"key": "1",
+			},
+			false,
+		},
+		{
+			"key=1.0",
+			map[string]interface{}{
+				"key": "1.0",
+			},
+			false,
+		},
+		{
+			"key=0x10",
+			map[string]interface{}{
+				"key": "0x10",
+			},
+			false,
+		},
+
+		// Test setting multiple times
+		{
+			[]string{
+				"foo=bar",
+				"bar=baz",
+			},
+			map[string]interface{}{
+				"foo": "bar",
+				"bar": "baz",
+			},
+			false,
+		},
+
+		// Test map merging
+		{
+			[]string{
+				`foo={ foo = "bar" }`,
+				`foo={ bar = "baz" }`,
+			},
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"foo": "bar",
+					"bar": "baz",
+				},
+			},
+			false,
+		},
+	}
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("%d-%s", i, tc.Input), func(t *testing.T) {
+			var input []string
+			switch v := tc.Input.(type) {
+			case string:
+				input = []string{v}
+			case []string:
+				input = v
+			default:
+				t.Fatalf("bad input type: %T", tc.Input)
+			}
+
+			f := new(FlagAny)
+			for i, single := range input {
+				err := f.Set(single)
+
+				// Only check for expected errors on the final input
+				expected := tc.Error && i == len(input)-1
+				if err != nil != expected {
+					t.Fatalf("bad error. Input: %#v\n\nError: %s", single, err)
+				}
+			}
+
+			actual := map[string]interface{}(*f)
+			if !reflect.DeepEqual(actual, tc.Output) {
+				t.Fatalf("bad:\nexpected: %s\n\n     got: %s\n", spew.Sdump(tc.Output), spew.Sdump(actual))
+			}
+		})
+	}
+}
+
+func TestFlagAny_file(t *testing.T) {
+	inputLibucl := `
+foo = "bar"
+`
+	inputMap := `
+foo = {
+	k = "v"
+}`
+
+	inputJson := `{
+		"foo": "bar"}`
+
+	cases := []struct {
+		Input  interface{}
+		Output map[string]interface{}
+		Error  bool
+	}{
+		{
+			inputLibucl,
+			map[string]interface{}{"foo": "bar"},
+			false,
+		},
+
+		{
+			inputJson,
+			map[string]interface{}{"foo": "bar"},
+			false,
+		},
+
+		{
+			`map.key = "foo"`,
+			map[string]interface{}{"map.key": "foo"},
+			false,
+		},
+
+		{
+			inputMap,
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"k": "v",
+				},
+			},
+			false,
+		},
+
+		{
+			[]string{
+				`foo = { "k" = "v"}`,
+				`foo = { "j" = "v" }`,
+			},
+			map[string]interface{}{
+				"foo": map[string]interface{}{
+					"k": "v",
+					"j": "v",
+				},
+			},
+			false,
+		},
+	}
+
+	path := testTempFile(t)
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var input []string
+			switch i := tc.Input.(type) {
+			case string:
+				input = []string{i}
+			case []string:
+				input = i
+			default:
+				t.Fatalf("bad input type: %T", i)
+			}
+
+			f := new(FlagAny)
+			for _, input := range input {
+				if err := ioutil.WriteFile(path, []byte(input), 0644); err != nil {
+					t.Fatalf("err: %s", err)
+				}
+
+				err := f.Set(path)
+				if err != nil != tc.Error {
+					t.Fatalf("bad error. Input: %#v, err: %s", input, err)
+				}
+			}
+
+			actual := map[string]interface{}(*f)
+			if !reflect.DeepEqual(actual, tc.Output) {
+				t.Fatalf("bad: %#v", actual)
+			}
+		})
+	}
+}

website/source/docs/backends/config.html.md

@@ -54,7 +54,7 @@ the configuration itself. We call this specifying only a _partial_ configuration
 
 With a partial configuration, the remaining configuration is expected as
 part of the [initialization](/docs/backends/init.html) process. There are
-two ways to supply the remaining configuration:
+a few ways to supply the remaining configuration:
 
   * **Interactively**: Terraform will interactively ask you for the required
     values. Terraform will not ask you for optional values.
@@ -63,13 +63,32 @@ two ways to supply the remaining configuration:
     This file can then be sourced via some secure means (such as
     [Vault](https://www.vaultproject.io)).
 
-In both cases, the final configuration is stored on disk in the
+  * **Command-line key/value pairs**: Key/value pairs in the format of
+    `key=value` can be specified as part of the init command. Note that
+    many shells retain command-line flags in a history file, so this isn't
+    recommended for secrets.
+
+In all cases, the final configuration is stored on disk in the
 ".terraform" directory, which should be ignored from version control.
 
 This means that sensitive information can be omitted from version control
 but it ultimately still lives on disk. In the future, Terraform may provide
 basic encryption on disk so that values are at least not plaintext.
 
+When using partial configuration, Terraform requires at a minimum that
+an empty backend configuration is in the Terraform files. For example:
+
+```
+terraform {
+  backend "consul" {}
+}
+```
+
+This minimal requirement allows Terraform to detect _unsetting_ backends.
+We cannot accept the backend type on the command-line because while it is
+technically possible, Terraform would then be unable to detect if you
+want to unset your backend (and move back to local state).
+
 ## Changing Configuration
 
 You can change your backend configuration at any time. You can change

website/source/docs/commands/init.html.markdown

@@ -44,17 +44,19 @@ The command-line flags are all optional. The list of available flags are:
 
 * `-backend=true` - Initialize the [backend](/docs/backends) for this environment.
 
-* `-backend-config=path` - Path to an HCL file with additional configuration
-  for the backend. This is merged with the backend in the Terraform configuration.
+* `-backend-config=value` - Value can be a path to an HCL file or a string
+  in the format of 'key=value'. This specifies additional configuration to merge
+  for the backend. This can be specified multiple times. Flags specified
+  later in the line override those specified earlier if they conflict.
 
 * `-get=true` - Download any modules for this configuration.
 
 * `-input=true` - Ask for input interactively if necessary. If this is false
   and input is required, `init` will error.
 
-## Backend Config File
+## Backend Config
 
-The `-backend-config` path can be used to specify additional
+The `-backend-config` can take a path or `key=value` pair to specify additional
 backend configuration when [initialize a backend](/docs/backends/init.html).
 
 This is particularly useful for
@@ -62,7 +64,7 @@ This is particularly useful for
 configuration lets you keep sensitive information out of your Terraform
 configuration.
 
-The backend configuration file is a basic HCL file with key/value pairs.
+For path values, the backend configuration file is a basic HCL file with key/value pairs.
 The keys are configuration keys for your backend. You do not need to wrap it
 in a `terraform` block. For example, the following file is a valid backend
 configuration file for the Consul backend type:
@@ -71,3 +73,17 @@ configuration file for the Consul backend type:
 address = "demo.consul.io"
 path    = "newpath"
 ```
+
+If the value contains an equal sign (`=`), it is parsed as a `key=value` pair.
+The format of this flag is identical to the `-var` flag for plan, apply,
+etc. but applies to configuration keys for backends. For example:
+
+```
+$ terraform init \
+  -backend-config 'address=demo.consul.io' \
+  -backend-config 'path=newpath'
+```
+
+These two formats can be mixed. In this case, the values will be merged by
+key with keys specified later in the command-line overriding conflicting
+keys specified earlier.