Merge pull request #18759 from hashicorp/b-terraform-remote-state

terraform/terraform_remote_state: accept complex configs

backend/remote-state/manta/backend.go

@@ -62,9 +62,17 @@ func New() backend.Backend {
 			},
 
 			"objectName": {
+				Type:       schema.TypeString,
+				Optional:   true,
+				Default:    "terraform.tfstate",
+				Deprecated: "please use the object_name attribute",
+			},
+
+			"object_name": {
 				Type:     schema.TypeString,
 				Optional: true,
-				Default:  "terraform.tfstate",
+				// Set this default once the objectName attribute is removed!
+				// Default:  "terraform.tfstate",
 			},
 		},
 	}
@@ -116,7 +124,12 @@ func (b *Backend) configure(ctx context.Context) error {
 	}
 
 	b.path = data.Get("path").(string)
-	b.objectName = data.Get("objectName").(string)
+	b.objectName = data.Get("object_name").(string)
+
+	// If object_name is not set, try the deprecated objectName.
+	if b.objectName == "" {
+		b.objectName = data.Get("objectName").(string)
+	}
 
 	var validationError *multierror.Error
 

backend/remote-state/manta/backend_test.go

@@ -31,8 +31,8 @@ func TestBackend(t *testing.T) {
 	keyName := "testState"
 
 	b := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	createMantaFolder(t, b.storageClient, directory)
@@ -48,13 +48,13 @@ func TestBackendLocked(t *testing.T) {
 	keyName := "testState"
 
 	b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	createMantaFolder(t, b1.storageClient, directory)
@@ -88,7 +88,6 @@ func deleteMantaFolder(t *testing.T, mantaClient *storage.StorageClient, directo
 	}
 
 	for _, obj := range objs.Entries {
-
 		if obj.Type == "directory" {
 			ojs, err := mantaClient.Dir().List(context.Background(), &storage.ListDirectoryInput{
 				DirectoryName: path.Join(mantaDefaultRootStore, directoryName, obj.Name),

backend/remote-state/manta/client_test.go

@@ -21,8 +21,8 @@ func TestRemoteClient(t *testing.T) {
 	keyName := "testState"
 
 	b := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	createMantaFolder(t, b.storageClient, directory)
@@ -42,13 +42,13 @@ func TestRemoteClientLocks(t *testing.T) {
 	keyName := "testState"
 
 	b1 := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	b2 := backend.TestBackendConfig(t, New(), map[string]interface{}{
-		"path":       directory,
+		"path":        directory,
-		"objectName": keyName,
+		"object_name": keyName,
 	}).(*Backend)
 
 	createMantaFolder(t, b1.storageClient, directory)

builtin/providers/terraform/data_source_state.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"time"
 
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/terraform/backend"
 	backendInit "github.com/hashicorp/terraform/backend/init"
 	"github.com/hashicorp/terraform/config"
@@ -32,9 +33,368 @@ func dataSourceRemoteState() *schema.Resource {
 				},
 			},
 
+			// This field now contains all possible attributes that are supported
+			// by any of the existing backends. When merging this into 0.12 this
+			// should be reverted and instead the new 'cty.DynamicPseudoType' type
+			// should be used to make this work with any future backends as well.
 			"config": {
-				Type:     schema.TypeMap,
+				Type:     schema.TypeSet,
 				Optional: true,
+				MaxItems: 1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"path": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"hostname": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"organization": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"token": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"workspaces": &schema.Schema{
+							Type:     schema.TypeList,
+							Optional: true,
+							MaxItems: 1,
+							Elem:     &schema.Schema{Type: schema.TypeMap},
+						},
+						"username": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"password": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"url": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"repo": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"subpath": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"storage_account_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"container_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"key": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"access_key": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"environment": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"resource_group_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"arm_subscription_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"arm_client_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"arm_client_secret": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"arm_tenant_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"access_token": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"address": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"scheme": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"datacenter": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"http_auth": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"gzip": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"lock": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"ca_file": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"cert_file": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"key_file": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"endpoints": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"prefix": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"cacert_path": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"cert_path": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"key_path": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"bucket": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"credentials": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"project": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"region": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"encryption_key": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"update_method": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"lock_address": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"lock_method": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"unlock_address": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"unlock_method": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_cert_verification": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"account": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"user": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"key_material": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"key_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"insecure_skip_tls_verify": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"object_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"endpoint": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"encrypt": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"acl": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"secret_key": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"kms_key_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"lock_table": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"dynamodb_table": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"profile": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"shared_credentials_file": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"role_arn": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"assume_role_policy": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"external_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"session_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"workspace_key_prefix": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_credentials_validation": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_get_ec2_platforms": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_region_validation": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_requesting_account_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"skip_metadata_api_check": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"auth_url": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"container": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"user_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"user_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"region_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"tenant_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"tenant_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"domain_id": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"domain_name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"insecure": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"cacert_file": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"cert": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"archive_container": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"archive_path": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"expire_after": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+						"name": &schema.Schema{
+							Type:     schema.TypeString,
+							Optional: true,
+						},
+					},
+				},
 			},
 
 			"defaults": {
@@ -66,8 +426,28 @@ func dataSourceRemoteState() *schema.Resource {
 func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
 	backendType := d.Get("backend").(string)
 
-	// Get the configuration in a type we want.
+	// Get the configuration in a type we want. This is a bit of a hack but makes
-	rawConfig, err := config.NewRawConfig(d.Get("config").(map[string]interface{}))
+	// things work for the 'remote' backend as well. This can simply be deleted or
+	// reverted when merging this 0.12.
+	raw := make(map[string]interface{})
+	if cfg, ok := d.GetOk("config"); ok {
+		if raw, ok = cfg.(*schema.Set).List()[0].(map[string]interface{}); ok {
+			for k, v := range raw {
+				switch v := v.(type) {
+				case string:
+					if v == "" {
+						delete(raw, k)
+					}
+				case []interface{}:
+					if len(v) == 0 {
+						delete(raw, k)
+					}
+				}
+			}
+		}
+	}
+
+	rawConfig, err := config.NewRawConfig(raw)
 	if err != nil {
 		return fmt.Errorf("error initializing backend: %s", err)
 	}
@@ -86,6 +466,14 @@ func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
 	}
 	b := f()
 
+	warns, errs := b.Validate(terraform.NewResourceConfig(rawConfig))
+	for _, warning := range warns {
+		log.Printf("[DEBUG] Warning validating backend config: %s", warning)
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("error validating backend config: %s", multierror.Append(nil, errs...))
+	}
+
 	// Configure the backend
 	if err := b.Configure(terraform.NewResourceConfig(rawConfig)); err != nil {
 		return fmt.Errorf("error initializing backend: %s", err)

builtin/providers/terraform/data_source_state_test.go

@@ -53,7 +53,9 @@ func TestState_complexOutputs(t *testing.T) {
 				Config: testAccState_complexOutputs,
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckStateValue("terraform_remote_state.foo", "backend", "local"),
-					testAccCheckStateValue("terraform_remote_state.foo", "config.path", "./test-fixtures/complex_outputs.tfstate"),
+					// This (adding the hash) should be reverted when merged into 0.12.
+					// testAccCheckStateValue("terraform_remote_state.foo", "config.path", "./test-fixtures/complex_outputs.tfstate"),
+					testAccCheckStateValue("terraform_remote_state.foo", "config.1590222752.path", "./test-fixtures/complex_outputs.tfstate"),
 					testAccCheckStateValue("terraform_remote_state.foo", "computed_set.#", "2"),
 					testAccCheckStateValue("terraform_remote_state.foo", `map.%`, "2"),
 					testAccCheckStateValue("terraform_remote_state.foo", `map.key`, "test"),

helper/schema/resource_data.go

@@ -315,6 +315,7 @@ func (d *ResourceData) State() *terraform.InstanceState {
 
 	mapW := &MapFieldWriter{Schema: d.schema}
 	if err := mapW.WriteField(nil, rawMap); err != nil {
+		log.Printf("[ERR] Error writing fields: %s", err)
 		return nil
 	}
 

website/docs/backends/types/manta.html.md

@@ -17,8 +17,8 @@ Stores the state as an artifact in [Manta](https://www.joyent.com/manta).
 ```hcl
 terraform {
   backend "manta" {
-    path       = "random/path"
+    path        = "random/path"
-    objectName = "terraform.tfstate"
+    object_name = "terraform.tfstate"
   }
 }
 ```
@@ -32,8 +32,8 @@ Note that for the access credentials we recommend using a
 data "terraform_remote_state" "foo" {
   backend = "manta"
   config {
-    path       = "random/path"
+    path        = "random/path"
-    objectName = "terraform.tfstate"
+    object_name = "terraform.tfstate"
   }
 }
 ```
@@ -49,4 +49,5 @@ The following configuration options are supported:
  * `key_id` - (Required) This is the fingerprint of the public key matching the key specified in key_path. It can be obtained via the command ssh-keygen -l -E md5 -f /path/to/key. Can be set via the `SDC_KEY_ID` or `TRITON_KEY_ID` environment variables.
  * `insecure_skip_tls_verify` - (Optional) This allows skipping TLS verification of the Triton endpoint. It is useful when connecting to a temporary Triton installation such as Cloud-On-A-Laptop which does not generally use a certificate signed by a trusted root CA. Defaults to `false`.
  * `path` - (Required) The path relative to your private storage directory (`/$MANTA_USER/stor`) where the state file will be stored. **Please Note:** If this path does not exist, then the backend will create this folder location as part of backend creation.
- * `objectName` - (Optional) The name of the state file (defaults to `terraform.tfstate`)
+ * `objectName` - (Optional, Deprecated) Use `object_name` instead.
+ * `object_name` - (Optional) The name of the state file (defaults to `terraform.tfstate`)