Add basic implementation for remote state on azure (#7064)

* Add basic implementation for remote state on azure

* Don't auto-provision the container

* Fix compilation errors

* Add factory to the remote map

* Add documentation

* Add acceptance tests

command/remote_config.go

@@ -348,8 +348,8 @@ Usage: terraform remote config [options]
 Options:
 
   -backend=Atlas         Specifies the type of remote backend. Must be one
-                         of Atlas, Consul, Etcd, GCS, HTTP, S3, or Swift. Defaults
+                         of Atlas, Consul, Etcd, GCS, HTTP, MAS, S3, or Swift.
-                         to Atlas.
+                         Defaults to Atlas.
 
   -backend-config="k=v"  Specifies configuration for the remote storage
                          backend. This can be specified multiple times.

state/remote/mas.go

@@ -0,0 +1,178 @@
+package remote
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/Azure/azure-sdk-for-go/arm/storage"
+	mainStorage "github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/Azure/go-autorest/autorest/azure"
+	riviera "github.com/jen20/riviera/azure"
+)
+
+func masFactory(conf map[string]string) (Client, error) {
+	storageAccountName, ok := conf["storage_account_name"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'storage_account_name' configuration")
+	}
+	containerName, ok := conf["container_name"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'container_name' configuration")
+	}
+	keyName, ok := conf["key"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'key' configuration")
+	}
+
+	accessKey, ok := confOrEnv(conf, "access_key", "ARM_ACCESS_KEY")
+	if !ok {
+		resourceGroupName, ok := conf["resource_group_name"]
+		if !ok {
+			return nil, fmt.Errorf("missing 'resource_group' configuration")
+		}
+
+		var err error
+		accessKey, err = getStorageAccountAccessKey(conf, resourceGroupName, storageAccountName)
+		if err != nil {
+			return nil, fmt.Errorf("Couldn't read access key from storage account: %s.", err)
+		}
+	}
+
+	storageClient, err := mainStorage.NewBasicClient(storageAccountName, accessKey)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating storage client for storage account %q: %s", storageAccountName, err)
+	}
+
+	blobClient := storageClient.GetBlobService()
+
+	return &MASClient{
+		blobClient:    &blobClient,
+		containerName: containerName,
+		keyName:       keyName,
+	}, nil
+}
+
+func getStorageAccountAccessKey(conf map[string]string, resourceGroupName, storageAccountName string) (string, error) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		return "", err
+	}
+
+	oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(creds.TenantID)
+	if err != nil {
+		return "", err
+	}
+	if oauthConfig == nil {
+		return "", fmt.Errorf("Unable to configure OAuthConfig for tenant %s", creds.TenantID)
+	}
+
+	spt, err := azure.NewServicePrincipalToken(*oauthConfig, creds.ClientID, creds.ClientSecret, azure.PublicCloud.ResourceManagerEndpoint)
+	if err != nil {
+		return "", err
+	}
+
+	accountsClient := storage.NewAccountsClient(creds.SubscriptionID)
+	accountsClient.Authorizer = spt
+
+	keys, err := accountsClient.ListKeys(resourceGroupName, storageAccountName)
+	if err != nil {
+		return "", fmt.Errorf("Error retrieving keys for storage account %q: %s", storageAccountName, err)
+	}
+
+	if keys.Key1 == nil {
+		return "", fmt.Errorf("Nil key returned for storage account %q", storageAccountName)
+	}
+
+	return *keys.Key1, nil
+}
+
+func getCredentialsFromConf(conf map[string]string) (*riviera.AzureResourceManagerCredentials, error) {
+	subscriptionID, ok := confOrEnv(conf, "arm_subscription_id", "ARM_SUBSCRIPTION_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_subscription_id' configuration")
+	}
+	clientID, ok := confOrEnv(conf, "arm_client_id", "ARM_CLIENT_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_client_id' configuration")
+	}
+	clientSecret, ok := confOrEnv(conf, "arm_client_secret", "ARM_CLIENT_SECRET")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_client_secret' configuration")
+	}
+	tenantID, ok := confOrEnv(conf, "arm_tenant_id", "ARM_TENANT_ID")
+	if !ok {
+		return nil, fmt.Errorf("missing 'arm_tenant_id' configuration")
+	}
+
+	return &riviera.AzureResourceManagerCredentials{
+		SubscriptionID: subscriptionID,
+		ClientID:       clientID,
+		ClientSecret:   clientSecret,
+		TenantID:       tenantID,
+	}, nil
+}
+
+func confOrEnv(conf map[string]string, confKey, envVar string) (string, bool) {
+	value, ok := conf[confKey]
+	if ok {
+		return value, true
+	}
+
+	value = os.Getenv(envVar)
+
+	return value, value != ""
+}
+
+type MASClient struct {
+	blobClient    *mainStorage.BlobStorageClient
+	containerName string
+	keyName       string
+}
+
+func (c *MASClient) Get() (*Payload, error) {
+	blob, err := c.blobClient.GetBlob(c.containerName, c.keyName)
+	if err != nil {
+		if storErr, ok := err.(mainStorage.AzureStorageServiceError); ok {
+			if storErr.Code == "BlobNotFound" {
+				return nil, nil
+			}
+		}
+		return nil, err
+	}
+
+	defer blob.Close()
+
+	data, err := ioutil.ReadAll(blob)
+	if err != nil {
+		return nil, err
+	}
+
+	payload := &Payload{
+		Data: data,
+	}
+
+	// If there was no data, then return nil
+	if len(payload.Data) == 0 {
+		return nil, nil
+	}
+
+	return payload, nil
+}
+
+func (c *MASClient) Put(data []byte) error {
+	return c.blobClient.CreateBlockBlobFromReader(
+		c.containerName,
+		c.keyName,
+		uint64(len(data)),
+		bytes.NewReader(data),
+		map[string]string{
+			"Content-Type": "application/json",
+		},
+	)
+}
+
+func (c *MASClient) Delete() error {
+	return c.blobClient.DeleteBlob(c.containerName, c.keyName, nil)
+}

state/remote/mas_test.go

@@ -0,0 +1,155 @@
+package remote
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	mainStorage "github.com/Azure/azure-sdk-for-go/storage"
+	riviera "github.com/jen20/riviera/azure"
+	"github.com/jen20/riviera/storage"
+)
+
+func TestMASClient_impl(t *testing.T) {
+	var _ Client = new(MASClient)
+}
+
+func TestMASClient(t *testing.T) {
+	// This test creates a bucket in MAS and populates it.
+	// It may incur costs, so it will only run if MAS credential environment
+	// variables are present.
+
+	config := map[string]string{
+		"arm_subscription_id": os.Getenv("ARM_SUBSCRIPTION_ID"),
+		"arm_client_id":       os.Getenv("ARM_CLIENT_ID"),
+		"arm_client_secret":   os.Getenv("ARM_CLIENT_SECRET"),
+		"arm_tenant_id":       os.Getenv("ARM_TENANT_ID"),
+	}
+
+	for k, v := range config {
+		if v == "" {
+			t.Skipf("skipping; %s must be set", strings.ToUpper(k))
+		}
+	}
+
+	config["resource_group_name"] = fmt.Sprintf("terraform-%x", time.Now().Unix())
+	config["storage_account_name"] = fmt.Sprintf("terraform%x", time.Now().Unix())
+	config["container_name"] = "terraform"
+	config["key"] = "test.tfstate"
+
+	setup(t, config)
+	defer teardown(t, config)
+
+	client, err := masFactory(config)
+	if err != nil {
+		t.Fatalf("Error for valid config: %v", err)
+	}
+
+	testClient(t, client)
+}
+
+func setup(t *testing.T, conf map[string]string) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		t.Fatalf("Error getting credentials from conf: %v", err)
+	}
+	rivieraClient, err := getRivieraClient(creds)
+	if err != nil {
+		t.Fatalf("Error instantiating the riviera client: %v", err)
+	}
+
+	// Create resource group
+	r := rivieraClient.NewRequest()
+	r.Command = riviera.CreateResourceGroup{
+		Name:     conf["resource_group_name"],
+		Location: riviera.WestUS,
+	}
+	response, err := r.Execute()
+	if err != nil {
+		t.Fatalf("Error creating a resource group: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error creating a resource group: %v", response.Error.Error())
+	}
+
+	// Create storage account
+	r = rivieraClient.NewRequest()
+	r.Command = storage.CreateStorageAccount{
+		ResourceGroupName: conf["resource_group_name"],
+		Name:              conf["storage_account_name"],
+		AccountType:       riviera.String("Standard_LRS"),
+		Location:          riviera.WestUS,
+	}
+	response, err = r.Execute()
+	if err != nil {
+		t.Fatalf("Error creating a storage account: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error creating a storage account: %v", response.Error.Error())
+	}
+
+	// Create container
+	accessKey, err := getStorageAccountAccessKey(conf, conf["resource_group_name"], conf["storage_account_name"])
+	if err != nil {
+		t.Fatalf("Error creating a storage account: %v", err)
+	}
+	storageClient, err := mainStorage.NewBasicClient(conf["storage_account_name"], accessKey)
+	if err != nil {
+		t.Fatalf("Error creating storage client for storage account %q: %s", conf["storage_account_name"], err)
+	}
+	blobClient := storageClient.GetBlobService()
+	_, err = blobClient.CreateContainerIfNotExists(conf["container_name"], mainStorage.ContainerAccessTypePrivate)
+	if err != nil {
+		t.Fatalf("Couldn't create container with name %s: %s.", conf["container_name"], err)
+	}
+}
+
+func teardown(t *testing.T, conf map[string]string) {
+	creds, err := getCredentialsFromConf(conf)
+	if err != nil {
+		t.Fatalf("Error getting credentials from conf: %v", err)
+	}
+	rivieraClient, err := getRivieraClient(creds)
+	if err != nil {
+		t.Fatalf("Error instantiating the riviera client: %v", err)
+	}
+
+	r := rivieraClient.NewRequest()
+	r.Command = riviera.DeleteResourceGroup{
+		Name: conf["resource_group_name"],
+	}
+	response, err := r.Execute()
+	if err != nil {
+		t.Fatalf("Error deleting the resource group: %v", err)
+	}
+	if !response.IsSuccessful() {
+		t.Fatalf("Error deleting the resource group: %v", err)
+	}
+}
+
+func getRivieraClient(credentials *riviera.AzureResourceManagerCredentials) (*riviera.Client, error) {
+	rivieraClient, err := riviera.NewClient(credentials)
+	if err != nil {
+		return nil, fmt.Errorf("Error creating Riviera client: %s", err)
+	}
+
+	request := rivieraClient.NewRequest()
+	request.Command = riviera.RegisterResourceProvider{
+		Namespace: "Microsoft.Storage",
+	}
+
+	response, err := request.Execute()
+	if err != nil {
+		return nil, fmt.Errorf("Cannot request provider registration for Azure Resource Manager: %s.", err)
+	}
+
+	if !response.IsSuccessful() {
+		return nil, fmt.Errorf("Credentials for acessing the Azure Resource Manager API are likely " +
+			"to be incorrect, or\n  the service principal does not have permission to use " +
+			"the Azure Service Management\n  API.")
+	}
+
+	return rivieraClient, nil
+}

state/remote/remote.go

@@ -41,6 +41,7 @@ var BuiltinClients = map[string]Factory{
 	"etcd":        etcdFactory,
 	"gcs":         gcsFactory,
 	"http":        httpFactory,
+	"mas":         masFactory,
 	"s3":          s3Factory,
 	"swift":       swiftFactory,
 	"artifactory": artifactoryFactory,

vendor/github.com/jen20/riviera/storage/api.go

@@ -0,0 +1,12 @@
+package storage
+
+import "fmt"
+
+const apiVersion = "2015-06-15"
+const apiProvider = "Microsoft.Storage"
+
+func storageDefaultURLPathFunc(resourceGroupName, storageAccountName string) func() string {
+	return func() string {
+		return fmt.Sprintf("resourceGroups/%s/providers/%s/storageAccounts/%s", resourceGroupName, apiProvider, storageAccountName)
+	}
+}

vendor/github.com/jen20/riviera/storage/create_storage_account.go

@@ -0,0 +1,27 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type CreateStorageAccountResponse struct {
+	Location    *string `mapstructure:"location"`
+	AccountType *string `mapstructure:"accountType"`
+}
+
+type CreateStorageAccount struct {
+	Name              string             `json:"-"`
+	ResourceGroupName string             `json:"-"`
+	AccountType       *string            `json:"accountType,omitempty"`
+	Location          string             `json:"-" riviera:"location"`
+	Tags              map[string]*string `json:"-" riviera:"tags"`
+}
+
+func (s CreateStorageAccount) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "PUT",
+		URLPathFunc: storageDefaultURLPathFunc(s.ResourceGroupName, s.Name),
+		ResponseTypeFunc: func() interface{} {
+			return &CreateStorageAccountResponse{}
+		},
+	}
+}

vendor/github.com/jen20/riviera/storage/delete_storage_account.go

@@ -0,0 +1,19 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type DeleteStorageAccount struct {
+	Name              string `json:"-"`
+	ResourceGroupName string `json:"-"`
+}
+
+func (command DeleteStorageAccount) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "DELETE",
+		URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
+		ResponseTypeFunc: func() interface{} {
+			return nil
+		},
+	}
+}

vendor/github.com/jen20/riviera/storage/get_storage_account_properties.go

@@ -0,0 +1,46 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type GetStorageAccountPropertiesResponse struct {
+	ID               *string `mapstructure:"id"`
+	Name             *string `mapstructure:"name"`
+	Location         *string `mapstructure:"location"`
+	AccountType      *string `mapstructure:"accountType"`
+	PrimaryEndpoints *struct {
+		Blob  *string `mapstructure:"blob"`
+		Queue *string `mapstructure:"queue"`
+		Table *string `mapstructure:"table"`
+		File  *string `mapstructure:"file"`
+	} `mapstructure:"primaryEndpoints"`
+	PrimaryLocation     *string `mapstructure:"primaryLocation"`
+	StatusOfPrimary     *string `mapstructure:"statusOfPrimary"`
+	LastGeoFailoverTime *string `mapstructure:"lastGeoFailoverTime"`
+	SecondaryLocation   *string `mapstructure:"secondaryLocation"`
+	StatusOfSecondary   *string `mapstructure:"statusOfSecondary"`
+	SecondaryEndpoints  *struct {
+		Blob  *string `mapstructure:"blob"`
+		Queue *string `mapstructure:"queue"`
+		Table *string `mapstructure:"table"`
+	} `mapstructure:"secondaryEndpoints"`
+	CreationTime *string `mapstructure:"creationTime"`
+	CustomDomain *struct {
+		Name *string `mapstructure:"name"`
+	} `mapstructure:"customDomain"`
+}
+
+type GetStorageAccountProperties struct {
+	Name              string `json:"-"`
+	ResourceGroupName string `json:"-"`
+}
+
+func (s GetStorageAccountProperties) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "GET",
+		URLPathFunc: storageDefaultURLPathFunc(s.ResourceGroupName, s.Name),
+		ResponseTypeFunc: func() interface{} {
+			return &GetStorageAccountPropertiesResponse{}
+		},
+	}
+}

vendor/github.com/jen20/riviera/storage/update_storage_account_custom_domain.go

@@ -0,0 +1,29 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type CustomDomain struct {
+	Name             *string `json:"name" mapstructure:"name"`
+	UseSubDomainName *bool   `json:"useSubDomainName,omitempty" mapstructure:"useSubdomainName"`
+}
+
+type UpdateStorageAccountCustomDomainResponse struct {
+	CustomDomain CustomDomain `mapstructure:"customDomain"`
+}
+
+type UpdateStorageAccountCustomDomain struct {
+	Name              string       `json:"-"`
+	ResourceGroupName string       `json:"-"`
+	CustomDomain      CustomDomain `json:"customDomain"`
+}
+
+func (command UpdateStorageAccountCustomDomain) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "PATCH",
+		URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
+		ResponseTypeFunc: func() interface{} {
+			return &UpdateStorageAccountCustomDomainResponse{}
+		},
+	}
+}

vendor/github.com/jen20/riviera/storage/update_storage_account_tags.go

@@ -0,0 +1,24 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type UpdateStorageAccountTagsResponse struct {
+	AccountType *string `mapstructure:"accountType"`
+}
+
+type UpdateStorageAccountTags struct {
+	Name              string             `json:"-"`
+	ResourceGroupName string             `json:"-"`
+	Tags              map[string]*string `json:"-" riviera:"tags"`
+}
+
+func (command UpdateStorageAccountTags) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "PATCH",
+		URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
+		ResponseTypeFunc: func() interface{} {
+			return &UpdateStorageAccountTypeResponse{}
+		},
+	}
+}

vendor/github.com/jen20/riviera/storage/update_storage_account_type.go

@@ -0,0 +1,24 @@
+package storage
+
+import "github.com/jen20/riviera/azure"
+
+type UpdateStorageAccountTypeResponse struct {
+	AccountType *string `mapstructure:"accountType"`
+}
+
+type UpdateStorageAccountType struct {
+	Name              string  `json:"-"`
+	ResourceGroupName string  `json:"-"`
+	AccountType       *string `json:"accountType,omitempty"`
+}
+
+func (command UpdateStorageAccountType) APIInfo() azure.APIInfo {
+	return azure.APIInfo{
+		APIVersion:  apiVersion,
+		Method:      "PATCH",
+		URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
+		ResponseTypeFunc: func() interface{} {
+			return &UpdateStorageAccountTypeResponse{}
+		},
+	}
+}

vendor/vendor.json

@@ -910,6 +910,12 @@
 			"path": "github.com/jen20/riviera/sql",
 			"revision": "70dac624f9d3e37295dfa4012040106e5f7b1add"
 		},
+		{
+			"checksumSHA1": "nKUCquNpJ9ifHgkXoT4K3Xar6R8=",
+			"path": "github.com/jen20/riviera/storage",
+			"revision": "64de55fa8cdd0c52f7d59494c1b03c1b583c52b4",
+			"revisionTime": "2016-02-18T23:50:40Z"
+		},
 		{
 			"comment": "0.2.2-2-gc01cf91",
 			"path": "github.com/jmespath/go-jmespath",

website/source/docs/state/remote/mas.html.md

@@ -0,0 +1,57 @@
+---
+layout: "remotestate"
+page_title: "Remote State Backend: mas"
+sidebar_current: "docs-state-remote-mas"
+description: |-
+  Terraform can store the state remotely, making it easier to version and work with in a team.
+---
+
+# mas
+
+Stores the state as a given key in a given bucket on [Microsoft Azure Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/).
+
+-> **Note:** Passing credentials directly via config options will
+make them included in cleartext inside the persisted state.
+Use of environment variables or config file is recommended.
+
+## Example Usage
+
+```
+terraform remote config \
+  -backend=mas \
+  -backend-config="storage_account_name=terraform123abc" \
+  -backend-config="container_name=terraform-state" \
+  -backend-config="key=prod.terraform.tfstate"
+```
+
+## Example Referencing
+
+```hcl
+# setup remote state data source
+data "terraform_remote_state" "foo" {
+  backend = "mas"
+  config {
+    storage_account_name = "terraform123abc"
+    container_name       = "terraform-state"
+    key                  = "prod.terraform.tfstate"
+  }
+}
+```
+
+## Configuration variables
+
+The following configuration options are supported:
+
+ * `storage_account_name` - (Required) The name of the storage account
+ * `container_name` - (Required) The name of the container to use within the storage account
+ * `key` - (Required) The key where to place/look for state file inside the container
+ * `access_key` / `ARM_ACCESS_KEY` - (Optional) Storage account access key
+ * `resource_group_name` - (Optional) The name of the resource group for the storage account. This is required when using the ARM credentials described below.
+ * `arm_subscription_id` - (Optional) The subscription ID to use. It can also
+  be sourced from the `ARM_SUBSCRIPTION_ID` environment variable.
+ * `arm_client_id` - (Optional) The client ID to use. It can also be sourced from
+  the `ARM_CLIENT_ID` environment variable.
+ * `arm_client_secret` - (Optional) The client secret to use. It can also be sourced from
+  the `ARM_CLIENT_SECRET` environment variable.
+ * `arm_tenant_id` - (Optional) The tenant ID to use. It can also be sourced from the
+  `ARM_TENANT_ID` environment variable.

website/source/layouts/remotestate.erb

@@ -16,6 +16,9 @@
 						<li<%= sidebar_current("docs-state-remote-artifactory") %>>
 							<a href="/docs/state/remote/artifactory.html">artifactory</a>
 						</li>
+						<li<%= sidebar_current("docs-state-remote-mas") %>>
+							<a href="/docs/state/remote/mas.html">mas</a>
+						</li>
 						<li<%= sidebar_current("docs-state-remote-atlas") %>>
 							<a href="/docs/state/remote/atlas.html">atlas</a>
 						</li>