Merge pull request #20480 from hashicorp/jbardin/backend-init

backend init defaults and inputs

backend/atlas/backend.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/configs/configschema"
-	"github.com/hashicorp/terraform/helper/schema"
 	"github.com/hashicorp/terraform/state"
 	"github.com/hashicorp/terraform/state/remote"
 	"github.com/hashicorp/terraform/terraform"
@@ -43,9 +42,6 @@ type Backend struct {
 	// stateClient is the legacy state client, setup in Configure
 	stateClient *stateClient
 
-	// schema is the schema for configuration, set by init
-	schema *schema.Backend
-
 	// opLock locks operations
 	opLock sync.Mutex
 }
@@ -79,7 +75,7 @@ func (b *Backend) ConfigSchema() *configschema.Block {
 	}
 }
 
-func (b *Backend) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
+func (b *Backend) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	name := obj.GetAttr("name").AsString()
@@ -105,7 +101,7 @@ func (b *Backend) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
 		}
 	}
 
-	return diags
+	return obj, diags
 }
 
 func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics {
@@ -116,7 +112,7 @@ func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics {
 		RunId: os.Getenv("ATLAS_RUN_ID"),
 	}
 
-	name := obj.GetAttr("name").AsString() // assumed valid due to ValidateConfig method
+	name := obj.GetAttr("name").AsString() // assumed valid due to PrepareConfig method
 	slashIdx := strings.Index(name, "/")
 	client.User = name[:slashIdx]
 	client.Name = name[slashIdx+1:]
@@ -139,7 +135,7 @@ func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics {
 		addr := v.AsString()
 		addrURL, err := url.Parse(addr)
 		if err != nil {
-			// We already validated the URL in ValidateConfig, so this shouldn't happen
+			// We already validated the URL in PrepareConfig, so this shouldn't happen
 			panic(err)
 		}
 		client.Server = addr

backend/backend.go

@@ -59,9 +59,10 @@ type Backend interface {
 	// be safely used before configuring.
 	ConfigSchema() *configschema.Block
 
-	// ValidateConfig checks the validity of the values in the given
-	// configuration, assuming that its structure has already been validated
-	// per the schema returned by ConfigSchema.
+	// PrepareConfig checks the validity of the values in the given
+	// configuration, and inserts any missing defaults, assuming that its
+	// structure has already been validated per the schema returned by
+	// ConfigSchema.
 	//
 	// This method does not have any side-effects for the backend and can
 	// be safely used before configuring. It also does not consult any
@@ -76,14 +77,14 @@ type Backend interface {
 	// as tfdiags.AttributeValue, and so the caller should provide the
 	// necessary context via the diags.InConfigBody method before returning
 	// diagnostics to the user.
-	ValidateConfig(cty.Value) tfdiags.Diagnostics
+	PrepareConfig(cty.Value) (cty.Value, tfdiags.Diagnostics)
 
 	// Configure uses the provided configuration to set configuration fields
 	// within the backend.
 	//
 	// The given configuration is assumed to have already been validated
 	// against the schema returned by ConfigSchema and passed validation
-	// via ValidateConfig.
+	// via PrepareConfig.
 	//
 	// This method may be called only once per backend instance, and must be
 	// called before all other methods except where otherwise stated.

backend/init/deprecate_test.go

@@ -14,7 +14,7 @@ func TestDeprecateBackend(t *testing.T) {
 		deprecateMessage,
 	)
 
-	diags := deprecatedBackend.ValidateConfig(cty.EmptyObjectVal)
+	_, diags := deprecatedBackend.PrepareConfig(cty.EmptyObjectVal)
 	if len(diags) != 1 {
 		t.Errorf("got %d diagnostics; want 1", len(diags))
 		for _, diag := range diags {

backend/init/init.go

@@ -108,11 +108,11 @@ type deprecatedBackendShim struct {
 	Message string
 }
 
-// ValidateConfig delegates to the wrapped backend to validate its config
+// PrepareConfig delegates to the wrapped backend to validate its config
 // and then appends shim's deprecation warning.
-func (b deprecatedBackendShim) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
-	diags := b.Backend.ValidateConfig(obj)
-	return diags.Append(tfdiags.SimpleWarning(b.Message))
+func (b deprecatedBackendShim) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) {
+	newObj, diags := b.Backend.PrepareConfig(obj)
+	return newObj, diags.Append(tfdiags.SimpleWarning(b.Message))
 }
 
 // DeprecateBackend can be used to wrap a backend to retrun a deprecation

backend/local/backend.go

@@ -139,9 +139,9 @@ func (b *Local) ConfigSchema() *configschema.Block {
 	}
 }
 
-func (b *Local) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
+func (b *Local) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) {
 	if b.Backend != nil {
-		return b.Backend.ValidateConfig(obj)
+		return b.Backend.PrepareConfig(obj)
 	}
 
 	var diags tfdiags.Diagnostics
@@ -170,7 +170,7 @@ func (b *Local) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
 		}
 	}
 
-	return diags
+	return obj, diags
 }
 
 func (b *Local) Configure(obj cty.Value) tfdiags.Diagnostics {

backend/nil.go

@@ -17,8 +17,8 @@ func (Nil) ConfigSchema() *configschema.Block {
 	return &configschema.Block{}
 }
 
-func (Nil) ValidateConfig(cty.Value) tfdiags.Diagnostics {
-	return nil
+func (Nil) PrepareConfig(v cty.Value) (cty.Value, tfdiags.Diagnostics) {
+	return v, nil
 }
 
 func (Nil) Configure(cty.Value) tfdiags.Diagnostics {

backend/remote-state/s3/backend.go

@@ -41,7 +41,10 @@ func New() backend.Backend {
 				Type:        schema.TypeString,
 				Required:    true,
 				Description: "The region of the S3 bucket.",
-				DefaultFunc: schema.EnvDefaultFunc("AWS_DEFAULT_REGION", nil),
+				DefaultFunc: schema.MultiEnvDefaultFunc([]string{
+					"AWS_REGION",
+					"AWS_DEFAULT_REGION",
+				}, nil),
 			},
 
 			"dynamodb_endpoint": {

backend/remote-state/s3/backend_test.go

@@ -76,7 +76,7 @@ func TestBackendConfig_invalidKey(t *testing.T) {
 		"dynamodb_table": "dynamoTable",
 	})
 
-	diags := New().ValidateConfig(cfg)
+	_, diags := New().PrepareConfig(cfg)
 	if !diags.HasErrors() {
 		t.Fatal("expected config validation error")
 	}

backend/remote/backend.go

@@ -139,8 +139,8 @@ func (b *Remote) ConfigSchema() *configschema.Block {
 	}
 }
 
-// ValidateConfig implements backend.Enhanced.
-func (b *Remote) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
+// PrepareConfig implements backend.Backend.
+func (b *Remote) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	if val := obj.GetAttr("organization"); val.IsNull() || val.AsString() == "" {
@@ -182,7 +182,7 @@ func (b *Remote) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
 		))
 	}
 
-	return diags
+	return obj, diags
 }
 
 // Configure implements backend.Enhanced.

backend/remote/backend_test.go

@@ -117,7 +117,7 @@ func TestRemote_config(t *testing.T) {
 		b := New(testDisco(s))
 
 		// Validate
-		valDiags := b.ValidateConfig(tc.config)
+		_, valDiags := b.PrepareConfig(tc.config)
 		if (valDiags.Err() != nil || tc.valErr != "") &&
 			(valDiags.Err() == nil || !strings.Contains(valDiags.Err().Error(), tc.valErr)) {
 			t.Fatalf("%s: unexpected validation result: %v", name, valDiags.Err())
@@ -196,7 +196,7 @@ func TestRemote_versionConstraints(t *testing.T) {
 		version.Version = tc.version
 
 		// Validate
-		valDiags := b.ValidateConfig(tc.config)
+		_, valDiags := b.PrepareConfig(tc.config)
 		if valDiags.HasErrors() {
 			t.Fatalf("%s: unexpected validation result: %v", name, valDiags.Err())
 		}

backend/remote/testing.go

@@ -97,10 +97,11 @@ func testBackend(t *testing.T, obj cty.Value) (*Remote, func()) {
 	b := New(testDisco(s))
 
 	// Configure the backend so the client is created.
-	valDiags := b.ValidateConfig(obj)
+	newObj, valDiags := b.PrepareConfig(obj)
 	if len(valDiags) != 0 {
 		t.Fatal(valDiags.ErrWithWarnings())
 	}
+	obj = newObj
 
 	confDiags := b.Configure(obj)
 	if len(confDiags) != 0 {

backend/testing.go

@@ -39,13 +39,15 @@ func TestBackendConfig(t *testing.T, b Backend, c hcl.Body) Backend {
 	obj, decDiags := hcldec.Decode(c, spec, nil)
 	diags = diags.Append(decDiags)
 
-	valDiags := b.ValidateConfig(obj)
+	newObj, valDiags := b.PrepareConfig(obj)
 	diags = diags.Append(valDiags.InConfigBody(c))
 
 	if len(diags) != 0 {
 		t.Fatal(diags.ErrWithWarnings())
 	}
 
+	obj = newObj
+
 	confDiags := b.Configure(obj)
 	if len(confDiags) != 0 {
 		confDiags = confDiags.InConfigBody(c)

builtin/providers/terraform/data_source_state.go

@@ -92,11 +92,12 @@ func dataSourceRemoteStateRead(d *cty.Value) (cty.Value, tfdiags.Diagnostics) {
 		return cty.NilVal, diags
 	}
 
-	validateDiags := b.ValidateConfig(configVal)
+	newVal, validateDiags := b.PrepareConfig(configVal)
 	diags = diags.Append(validateDiags)
 	if validateDiags.HasErrors() {
 		return cty.NilVal, diags
 	}
+	configVal = newVal
 
 	configureDiags := b.Configure(configVal)
 	if configureDiags.HasErrors() {

command/init.go

@@ -446,6 +446,7 @@ func (c *InitCommand) initBackend(root *configs.Module, extraConfig rawFlags) (b
 		ConfigOverride: backendConfigOverride,
 		Init:           true,
 	}
+
 	back, backDiags := c.Backend(opts)
 	diags = diags.Append(backDiags)
 	return back, true, diags

command/meta_backend.go

@@ -180,11 +180,12 @@ func (m *Meta) BackendForPlan(settings plans.Backend) (backend.Enhanced, tfdiags
 		return nil, diags
 	}
 
-	validateDiags := b.ValidateConfig(configVal)
+	newVal, validateDiags := b.PrepareConfig(configVal)
 	diags = diags.Append(validateDiags)
 	if validateDiags.HasErrors() {
 		return nil, diags
 	}
+	configVal = newVal
 
 	configureDiags := b.Configure(configVal)
 	diags = diags.Append(configureDiags)
@@ -917,11 +918,13 @@ func (m *Meta) backend_C_r_S_unchanged(c *configs.Backend, cHash int, sMgr *stat
 	}
 
 	// Validate the config and then configure the backend
-	validDiags := b.ValidateConfig(configVal)
+	newVal, validDiags := b.PrepareConfig(configVal)
 	diags = diags.Append(validDiags)
 	if validDiags.HasErrors() {
 		return nil, diags
 	}
+	configVal = newVal
+
 	configDiags := b.Configure(configVal)
 	diags = diags.Append(configDiags)
 	if configDiags.HasErrors() {
@@ -1037,11 +1040,12 @@ func (m *Meta) backendInitFromConfig(c *configs.Backend) (backend.Backend, cty.V
 		}
 	}
 
-	validateDiags := b.ValidateConfig(configVal)
+	newVal, validateDiags := b.PrepareConfig(configVal)
 	diags = diags.Append(validateDiags.InConfigBody(c.Config))
 	if validateDiags.HasErrors() {
 		return nil, cty.NilVal, diags
 	}
+	configVal = newVal
 
 	configureDiags := b.Configure(configVal)
 	diags = diags.Append(configureDiags.InConfigBody(c.Config))
@@ -1067,11 +1071,12 @@ func (m *Meta) backendInitFromSaved(s *terraform.BackendState) (backend.Backend,
 		return nil, diags
 	}
 
-	validateDiags := b.ValidateConfig(configVal)
+	newVal, validateDiags := b.PrepareConfig(configVal)
 	diags = diags.Append(validateDiags)
 	if validateDiags.HasErrors() {
 		return nil, diags
 	}
+	configVal = newVal
 
 	configureDiags := b.Configure(configVal)
 	diags = diags.Append(configureDiags)

helper/schema/backend.go

@@ -2,6 +2,7 @@ package schema
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/zclconf/go-cty/cty"
@@ -9,6 +10,7 @@ import (
 	"github.com/hashicorp/terraform/config/hcl2shim"
 	"github.com/hashicorp/terraform/configs/configschema"
 	"github.com/hashicorp/terraform/terraform"
+	ctyconvert "github.com/zclconf/go-cty/cty/convert"
 )
 
 // Backend represents a partial backend.Backend implementation and simplifies
@@ -49,13 +51,86 @@ func (b *Backend) ConfigSchema() *configschema.Block {
 	return b.CoreConfigSchema()
 }
 
-func (b *Backend) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
+func (b *Backend) PrepareConfig(configVal cty.Value) (cty.Value, tfdiags.Diagnostics) {
 	if b == nil {
-		return nil
+		return configVal, nil
+	}
+	var diags tfdiags.Diagnostics
+	var err error
+
+	// In order to use Transform below, this needs to be filled out completely
+	// according the schema.
+	configVal, err = b.CoreConfigSchema().CoerceValue(configVal)
+	if err != nil {
+		return configVal, diags.Append(err)
 	}
 
-	var diags tfdiags.Diagnostics
-	shimRC := b.shimConfig(obj)
+	// lookup any required, top-level attributes that are Null, and see if we
+	// have a Default value available.
+	configVal, err = cty.Transform(configVal, func(path cty.Path, val cty.Value) (cty.Value, error) {
+		// we're only looking for top-level attributes
+		if len(path) != 1 {
+			return val, nil
+		}
+
+		// nothing to do if we already have a value
+		if !val.IsNull() {
+			return val, nil
+		}
+
+		// get the Schema definition for this attribute
+		getAttr, ok := path[0].(cty.GetAttrStep)
+		// these should all exist, but just ignore anything strange
+		if !ok {
+			return val, nil
+		}
+
+		attrSchema := b.Schema[getAttr.Name]
+		// continue to ignore anything that doesn't match
+		if attrSchema == nil {
+			return val, nil
+		}
+
+		// this is deprecated, so don't set it
+		if attrSchema.Deprecated != "" || attrSchema.Removed != "" {
+			return val, nil
+		}
+
+		// find a default value if it exists
+		def, err := attrSchema.DefaultValue()
+		if err != nil {
+			diags = diags.Append(fmt.Errorf("error getting default for %q: %s", getAttr.Name, err))
+			return val, err
+		}
+
+		// no default
+		if def == nil {
+			return val, nil
+		}
+
+		// create a cty.Value and make sure it's the correct type
+		tmpVal := hcl2shim.HCL2ValueFromConfigValue(def)
+
+		// helper/schema used to allow setting "" to a bool
+		if val.Type() == cty.Bool && tmpVal.RawEquals(cty.StringVal("")) {
+			// return a warning about the conversion
+			diags = diags.Append("provider set empty string as default value for bool " + getAttr.Name)
+			tmpVal = cty.False
+		}
+
+		val, err = ctyconvert.Convert(tmpVal, val.Type())
+		if err != nil {
+			diags = diags.Append(fmt.Errorf("error setting default for %q: %s", getAttr.Name, err))
+		}
+
+		return val, err
+	})
+	if err != nil {
+		// any error here was already added to the diagnostics
+		return configVal, diags
+	}
+
+	shimRC := b.shimConfig(configVal)
 	warns, errs := schemaMap(b.Schema).Validate(shimRC)
 	for _, warn := range warns {
 		diags = diags.Append(tfdiags.SimpleWarning(warn))
@@ -63,7 +138,7 @@ func (b *Backend) ValidateConfig(obj cty.Value) tfdiags.Diagnostics {
 	for _, err := range errs {
 		diags = diags.Append(err)
 	}
-	return diags
+	return configVal, diags
 }
 
 func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics {

helper/schema/backend_test.go

@@ -8,11 +8,12 @@ import (
 	"github.com/zclconf/go-cty/cty"
 )
 
-func TestBackendValidate(t *testing.T) {
+func TestBackendPrepare(t *testing.T) {
 	cases := []struct {
 		Name   string
 		B      *Backend
 		Config map[string]cty.Value
+		Expect map[string]cty.Value
 		Err    bool
 	}{
 		{
@@ -26,6 +27,7 @@ func TestBackendValidate(t *testing.T) {
 				},
 			},
 			map[string]cty.Value{},
+			map[string]cty.Value{},
 			true,
 		},
 
@@ -42,15 +44,87 @@ func TestBackendValidate(t *testing.T) {
 			map[string]cty.Value{
 				"foo": cty.StringVal("bar"),
 			},
+			map[string]cty.Value{
+				"foo": cty.StringVal("bar"),
+			},
+			false,
+		},
+
+		{
+			"unused default",
+			&Backend{
+				Schema: map[string]*Schema{
+					"foo": &Schema{
+						Optional: true,
+						Type:     TypeString,
+						Default:  "baz",
+					},
+				},
+			},
+			map[string]cty.Value{
+				"foo": cty.StringVal("bar"),
+			},
+			map[string]cty.Value{
+				"foo": cty.StringVal("bar"),
+			},
+			false,
+		},
+
+		{
+			"default",
+			&Backend{
+				Schema: map[string]*Schema{
+					"foo": &Schema{
+						Type:     TypeString,
+						Optional: true,
+						Default:  "baz",
+					},
+				},
+			},
+			map[string]cty.Value{},
+			map[string]cty.Value{
+				"foo": cty.StringVal("baz"),
+			},
+			false,
+		},
+
+		{
+			"default func",
+			&Backend{
+				Schema: map[string]*Schema{
+					"foo": &Schema{
+						Type:     TypeString,
+						Optional: true,
+						DefaultFunc: func() (interface{}, error) {
+							return "baz", nil
+						},
+					},
+				},
+			},
+			map[string]cty.Value{},
+			map[string]cty.Value{
+				"foo": cty.StringVal("baz"),
+			},
 			false,
 		},
 	}
 
 	for i, tc := range cases {
 		t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
-			diags := tc.B.ValidateConfig(cty.ObjectVal(tc.Config))
+			configVal, diags := tc.B.PrepareConfig(cty.ObjectVal(tc.Config))
 			if diags.HasErrors() != tc.Err {
-				t.Errorf("wrong number of diagnostics")
+				for _, d := range diags {
+					t.Error(d.Description())
+				}
+			}
+
+			if tc.Err {
+				return
+			}
+
+			expect := cty.ObjectVal(tc.Expect)
+			if !expect.RawEquals(configVal) {
+				t.Fatalf("\nexpected: %#v\ngot:     %#v\n", expect, configVal)
 			}
 		})
 	}