Add basic implementation for remote state on azure (#7064)

* Add basic implementation for remote state on azure

* Don't auto-provision the container

* Fix compilation errors

* Add factory to the remote map

* Add documentation

* Add acceptance tests
This commit is contained in:
Maxime Bury 2016-06-10 10:27:57 -07:00 committed by Paul Stack
parent 511101ab75
commit c98f391bee
14 changed files with 583 additions and 2 deletions

View File

@ -348,8 +348,8 @@ Usage: terraform remote config [options]
Options:
-backend=Atlas Specifies the type of remote backend. Must be one
of Atlas, Consul, Etcd, GCS, HTTP, S3, or Swift. Defaults
to Atlas.
of Atlas, Consul, Etcd, GCS, HTTP, MAS, S3, or Swift.
Defaults to Atlas.
-backend-config="k=v" Specifies configuration for the remote storage
backend. This can be specified multiple times.

178
state/remote/mas.go Normal file
View File

@ -0,0 +1,178 @@
package remote
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"github.com/Azure/azure-sdk-for-go/arm/storage"
mainStorage "github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
riviera "github.com/jen20/riviera/azure"
)
func masFactory(conf map[string]string) (Client, error) {
storageAccountName, ok := conf["storage_account_name"]
if !ok {
return nil, fmt.Errorf("missing 'storage_account_name' configuration")
}
containerName, ok := conf["container_name"]
if !ok {
return nil, fmt.Errorf("missing 'container_name' configuration")
}
keyName, ok := conf["key"]
if !ok {
return nil, fmt.Errorf("missing 'key' configuration")
}
accessKey, ok := confOrEnv(conf, "access_key", "ARM_ACCESS_KEY")
if !ok {
resourceGroupName, ok := conf["resource_group_name"]
if !ok {
return nil, fmt.Errorf("missing 'resource_group' configuration")
}
var err error
accessKey, err = getStorageAccountAccessKey(conf, resourceGroupName, storageAccountName)
if err != nil {
return nil, fmt.Errorf("Couldn't read access key from storage account: %s.", err)
}
}
storageClient, err := mainStorage.NewBasicClient(storageAccountName, accessKey)
if err != nil {
return nil, fmt.Errorf("Error creating storage client for storage account %q: %s", storageAccountName, err)
}
blobClient := storageClient.GetBlobService()
return &MASClient{
blobClient: &blobClient,
containerName: containerName,
keyName: keyName,
}, nil
}
func getStorageAccountAccessKey(conf map[string]string, resourceGroupName, storageAccountName string) (string, error) {
creds, err := getCredentialsFromConf(conf)
if err != nil {
return "", err
}
oauthConfig, err := azure.PublicCloud.OAuthConfigForTenant(creds.TenantID)
if err != nil {
return "", err
}
if oauthConfig == nil {
return "", fmt.Errorf("Unable to configure OAuthConfig for tenant %s", creds.TenantID)
}
spt, err := azure.NewServicePrincipalToken(*oauthConfig, creds.ClientID, creds.ClientSecret, azure.PublicCloud.ResourceManagerEndpoint)
if err != nil {
return "", err
}
accountsClient := storage.NewAccountsClient(creds.SubscriptionID)
accountsClient.Authorizer = spt
keys, err := accountsClient.ListKeys(resourceGroupName, storageAccountName)
if err != nil {
return "", fmt.Errorf("Error retrieving keys for storage account %q: %s", storageAccountName, err)
}
if keys.Key1 == nil {
return "", fmt.Errorf("Nil key returned for storage account %q", storageAccountName)
}
return *keys.Key1, nil
}
func getCredentialsFromConf(conf map[string]string) (*riviera.AzureResourceManagerCredentials, error) {
subscriptionID, ok := confOrEnv(conf, "arm_subscription_id", "ARM_SUBSCRIPTION_ID")
if !ok {
return nil, fmt.Errorf("missing 'arm_subscription_id' configuration")
}
clientID, ok := confOrEnv(conf, "arm_client_id", "ARM_CLIENT_ID")
if !ok {
return nil, fmt.Errorf("missing 'arm_client_id' configuration")
}
clientSecret, ok := confOrEnv(conf, "arm_client_secret", "ARM_CLIENT_SECRET")
if !ok {
return nil, fmt.Errorf("missing 'arm_client_secret' configuration")
}
tenantID, ok := confOrEnv(conf, "arm_tenant_id", "ARM_TENANT_ID")
if !ok {
return nil, fmt.Errorf("missing 'arm_tenant_id' configuration")
}
return &riviera.AzureResourceManagerCredentials{
SubscriptionID: subscriptionID,
ClientID: clientID,
ClientSecret: clientSecret,
TenantID: tenantID,
}, nil
}
func confOrEnv(conf map[string]string, confKey, envVar string) (string, bool) {
value, ok := conf[confKey]
if ok {
return value, true
}
value = os.Getenv(envVar)
return value, value != ""
}
type MASClient struct {
blobClient *mainStorage.BlobStorageClient
containerName string
keyName string
}
func (c *MASClient) Get() (*Payload, error) {
blob, err := c.blobClient.GetBlob(c.containerName, c.keyName)
if err != nil {
if storErr, ok := err.(mainStorage.AzureStorageServiceError); ok {
if storErr.Code == "BlobNotFound" {
return nil, nil
}
}
return nil, err
}
defer blob.Close()
data, err := ioutil.ReadAll(blob)
if err != nil {
return nil, err
}
payload := &Payload{
Data: data,
}
// If there was no data, then return nil
if len(payload.Data) == 0 {
return nil, nil
}
return payload, nil
}
func (c *MASClient) Put(data []byte) error {
return c.blobClient.CreateBlockBlobFromReader(
c.containerName,
c.keyName,
uint64(len(data)),
bytes.NewReader(data),
map[string]string{
"Content-Type": "application/json",
},
)
}
func (c *MASClient) Delete() error {
return c.blobClient.DeleteBlob(c.containerName, c.keyName, nil)
}

155
state/remote/mas_test.go Normal file
View File

@ -0,0 +1,155 @@
package remote
import (
"fmt"
"os"
"strings"
"testing"
"time"
mainStorage "github.com/Azure/azure-sdk-for-go/storage"
riviera "github.com/jen20/riviera/azure"
"github.com/jen20/riviera/storage"
)
func TestMASClient_impl(t *testing.T) {
var _ Client = new(MASClient)
}
func TestMASClient(t *testing.T) {
// This test creates a bucket in MAS and populates it.
// It may incur costs, so it will only run if MAS credential environment
// variables are present.
config := map[string]string{
"arm_subscription_id": os.Getenv("ARM_SUBSCRIPTION_ID"),
"arm_client_id": os.Getenv("ARM_CLIENT_ID"),
"arm_client_secret": os.Getenv("ARM_CLIENT_SECRET"),
"arm_tenant_id": os.Getenv("ARM_TENANT_ID"),
}
for k, v := range config {
if v == "" {
t.Skipf("skipping; %s must be set", strings.ToUpper(k))
}
}
config["resource_group_name"] = fmt.Sprintf("terraform-%x", time.Now().Unix())
config["storage_account_name"] = fmt.Sprintf("terraform%x", time.Now().Unix())
config["container_name"] = "terraform"
config["key"] = "test.tfstate"
setup(t, config)
defer teardown(t, config)
client, err := masFactory(config)
if err != nil {
t.Fatalf("Error for valid config: %v", err)
}
testClient(t, client)
}
func setup(t *testing.T, conf map[string]string) {
creds, err := getCredentialsFromConf(conf)
if err != nil {
t.Fatalf("Error getting credentials from conf: %v", err)
}
rivieraClient, err := getRivieraClient(creds)
if err != nil {
t.Fatalf("Error instantiating the riviera client: %v", err)
}
// Create resource group
r := rivieraClient.NewRequest()
r.Command = riviera.CreateResourceGroup{
Name: conf["resource_group_name"],
Location: riviera.WestUS,
}
response, err := r.Execute()
if err != nil {
t.Fatalf("Error creating a resource group: %v", err)
}
if !response.IsSuccessful() {
t.Fatalf("Error creating a resource group: %v", response.Error.Error())
}
// Create storage account
r = rivieraClient.NewRequest()
r.Command = storage.CreateStorageAccount{
ResourceGroupName: conf["resource_group_name"],
Name: conf["storage_account_name"],
AccountType: riviera.String("Standard_LRS"),
Location: riviera.WestUS,
}
response, err = r.Execute()
if err != nil {
t.Fatalf("Error creating a storage account: %v", err)
}
if !response.IsSuccessful() {
t.Fatalf("Error creating a storage account: %v", response.Error.Error())
}
// Create container
accessKey, err := getStorageAccountAccessKey(conf, conf["resource_group_name"], conf["storage_account_name"])
if err != nil {
t.Fatalf("Error creating a storage account: %v", err)
}
storageClient, err := mainStorage.NewBasicClient(conf["storage_account_name"], accessKey)
if err != nil {
t.Fatalf("Error creating storage client for storage account %q: %s", conf["storage_account_name"], err)
}
blobClient := storageClient.GetBlobService()
_, err = blobClient.CreateContainerIfNotExists(conf["container_name"], mainStorage.ContainerAccessTypePrivate)
if err != nil {
t.Fatalf("Couldn't create container with name %s: %s.", conf["container_name"], err)
}
}
func teardown(t *testing.T, conf map[string]string) {
creds, err := getCredentialsFromConf(conf)
if err != nil {
t.Fatalf("Error getting credentials from conf: %v", err)
}
rivieraClient, err := getRivieraClient(creds)
if err != nil {
t.Fatalf("Error instantiating the riviera client: %v", err)
}
r := rivieraClient.NewRequest()
r.Command = riviera.DeleteResourceGroup{
Name: conf["resource_group_name"],
}
response, err := r.Execute()
if err != nil {
t.Fatalf("Error deleting the resource group: %v", err)
}
if !response.IsSuccessful() {
t.Fatalf("Error deleting the resource group: %v", err)
}
}
func getRivieraClient(credentials *riviera.AzureResourceManagerCredentials) (*riviera.Client, error) {
rivieraClient, err := riviera.NewClient(credentials)
if err != nil {
return nil, fmt.Errorf("Error creating Riviera client: %s", err)
}
request := rivieraClient.NewRequest()
request.Command = riviera.RegisterResourceProvider{
Namespace: "Microsoft.Storage",
}
response, err := request.Execute()
if err != nil {
return nil, fmt.Errorf("Cannot request provider registration for Azure Resource Manager: %s.", err)
}
if !response.IsSuccessful() {
return nil, fmt.Errorf("Credentials for acessing the Azure Resource Manager API are likely " +
"to be incorrect, or\n the service principal does not have permission to use " +
"the Azure Service Management\n API.")
}
return rivieraClient, nil
}

View File

@ -41,6 +41,7 @@ var BuiltinClients = map[string]Factory{
"etcd": etcdFactory,
"gcs": gcsFactory,
"http": httpFactory,
"mas": masFactory,
"s3": s3Factory,
"swift": swiftFactory,
"artifactory": artifactoryFactory,

12
vendor/github.com/jen20/riviera/storage/api.go generated vendored Normal file
View File

@ -0,0 +1,12 @@
package storage
import "fmt"
const apiVersion = "2015-06-15"
const apiProvider = "Microsoft.Storage"
func storageDefaultURLPathFunc(resourceGroupName, storageAccountName string) func() string {
return func() string {
return fmt.Sprintf("resourceGroups/%s/providers/%s/storageAccounts/%s", resourceGroupName, apiProvider, storageAccountName)
}
}

View File

@ -0,0 +1,27 @@
package storage
import "github.com/jen20/riviera/azure"
type CreateStorageAccountResponse struct {
Location *string `mapstructure:"location"`
AccountType *string `mapstructure:"accountType"`
}
type CreateStorageAccount struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
AccountType *string `json:"accountType,omitempty"`
Location string `json:"-" riviera:"location"`
Tags map[string]*string `json:"-" riviera:"tags"`
}
func (s CreateStorageAccount) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "PUT",
URLPathFunc: storageDefaultURLPathFunc(s.ResourceGroupName, s.Name),
ResponseTypeFunc: func() interface{} {
return &CreateStorageAccountResponse{}
},
}
}

View File

@ -0,0 +1,19 @@
package storage
import "github.com/jen20/riviera/azure"
type DeleteStorageAccount struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
}
func (command DeleteStorageAccount) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "DELETE",
URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
ResponseTypeFunc: func() interface{} {
return nil
},
}
}

View File

@ -0,0 +1,46 @@
package storage
import "github.com/jen20/riviera/azure"
type GetStorageAccountPropertiesResponse struct {
ID *string `mapstructure:"id"`
Name *string `mapstructure:"name"`
Location *string `mapstructure:"location"`
AccountType *string `mapstructure:"accountType"`
PrimaryEndpoints *struct {
Blob *string `mapstructure:"blob"`
Queue *string `mapstructure:"queue"`
Table *string `mapstructure:"table"`
File *string `mapstructure:"file"`
} `mapstructure:"primaryEndpoints"`
PrimaryLocation *string `mapstructure:"primaryLocation"`
StatusOfPrimary *string `mapstructure:"statusOfPrimary"`
LastGeoFailoverTime *string `mapstructure:"lastGeoFailoverTime"`
SecondaryLocation *string `mapstructure:"secondaryLocation"`
StatusOfSecondary *string `mapstructure:"statusOfSecondary"`
SecondaryEndpoints *struct {
Blob *string `mapstructure:"blob"`
Queue *string `mapstructure:"queue"`
Table *string `mapstructure:"table"`
} `mapstructure:"secondaryEndpoints"`
CreationTime *string `mapstructure:"creationTime"`
CustomDomain *struct {
Name *string `mapstructure:"name"`
} `mapstructure:"customDomain"`
}
type GetStorageAccountProperties struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
}
func (s GetStorageAccountProperties) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "GET",
URLPathFunc: storageDefaultURLPathFunc(s.ResourceGroupName, s.Name),
ResponseTypeFunc: func() interface{} {
return &GetStorageAccountPropertiesResponse{}
},
}
}

View File

@ -0,0 +1,29 @@
package storage
import "github.com/jen20/riviera/azure"
type CustomDomain struct {
Name *string `json:"name" mapstructure:"name"`
UseSubDomainName *bool `json:"useSubDomainName,omitempty" mapstructure:"useSubdomainName"`
}
type UpdateStorageAccountCustomDomainResponse struct {
CustomDomain CustomDomain `mapstructure:"customDomain"`
}
type UpdateStorageAccountCustomDomain struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
CustomDomain CustomDomain `json:"customDomain"`
}
func (command UpdateStorageAccountCustomDomain) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "PATCH",
URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
ResponseTypeFunc: func() interface{} {
return &UpdateStorageAccountCustomDomainResponse{}
},
}
}

View File

@ -0,0 +1,24 @@
package storage
import "github.com/jen20/riviera/azure"
type UpdateStorageAccountTagsResponse struct {
AccountType *string `mapstructure:"accountType"`
}
type UpdateStorageAccountTags struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
Tags map[string]*string `json:"-" riviera:"tags"`
}
func (command UpdateStorageAccountTags) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "PATCH",
URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
ResponseTypeFunc: func() interface{} {
return &UpdateStorageAccountTypeResponse{}
},
}
}

View File

@ -0,0 +1,24 @@
package storage
import "github.com/jen20/riviera/azure"
type UpdateStorageAccountTypeResponse struct {
AccountType *string `mapstructure:"accountType"`
}
type UpdateStorageAccountType struct {
Name string `json:"-"`
ResourceGroupName string `json:"-"`
AccountType *string `json:"accountType,omitempty"`
}
func (command UpdateStorageAccountType) APIInfo() azure.APIInfo {
return azure.APIInfo{
APIVersion: apiVersion,
Method: "PATCH",
URLPathFunc: storageDefaultURLPathFunc(command.ResourceGroupName, command.Name),
ResponseTypeFunc: func() interface{} {
return &UpdateStorageAccountTypeResponse{}
},
}
}

6
vendor/vendor.json vendored
View File

@ -910,6 +910,12 @@
"path": "github.com/jen20/riviera/sql",
"revision": "70dac624f9d3e37295dfa4012040106e5f7b1add"
},
{
"checksumSHA1": "nKUCquNpJ9ifHgkXoT4K3Xar6R8=",
"path": "github.com/jen20/riviera/storage",
"revision": "64de55fa8cdd0c52f7d59494c1b03c1b583c52b4",
"revisionTime": "2016-02-18T23:50:40Z"
},
{
"comment": "0.2.2-2-gc01cf91",
"path": "github.com/jmespath/go-jmespath",

View File

@ -0,0 +1,57 @@
---
layout: "remotestate"
page_title: "Remote State Backend: mas"
sidebar_current: "docs-state-remote-mas"
description: |-
Terraform can store the state remotely, making it easier to version and work with in a team.
---
# mas
Stores the state as a given key in a given bucket on [Microsoft Azure Storage](https://azure.microsoft.com/en-us/documentation/articles/storage-introduction/).
-> **Note:** Passing credentials directly via config options will
make them included in cleartext inside the persisted state.
Use of environment variables or config file is recommended.
## Example Usage
```
terraform remote config \
-backend=mas \
-backend-config="storage_account_name=terraform123abc" \
-backend-config="container_name=terraform-state" \
-backend-config="key=prod.terraform.tfstate"
```
## Example Referencing
```hcl
# setup remote state data source
data "terraform_remote_state" "foo" {
backend = "mas"
config {
storage_account_name = "terraform123abc"
container_name = "terraform-state"
key = "prod.terraform.tfstate"
}
}
```
## Configuration variables
The following configuration options are supported:
* `storage_account_name` - (Required) The name of the storage account
* `container_name` - (Required) The name of the container to use within the storage account
* `key` - (Required) The key where to place/look for state file inside the container
* `access_key` / `ARM_ACCESS_KEY` - (Optional) Storage account access key
* `resource_group_name` - (Optional) The name of the resource group for the storage account. This is required when using the ARM credentials described below.
* `arm_subscription_id` - (Optional) The subscription ID to use. It can also
be sourced from the `ARM_SUBSCRIPTION_ID` environment variable.
* `arm_client_id` - (Optional) The client ID to use. It can also be sourced from
the `ARM_CLIENT_ID` environment variable.
* `arm_client_secret` - (Optional) The client secret to use. It can also be sourced from
the `ARM_CLIENT_SECRET` environment variable.
* `arm_tenant_id` - (Optional) The tenant ID to use. It can also be sourced from the
`ARM_TENANT_ID` environment variable.

View File

@ -16,6 +16,9 @@
<li<%= sidebar_current("docs-state-remote-artifactory") %>>
<a href="/docs/state/remote/artifactory.html">artifactory</a>
</li>
<li<%= sidebar_current("docs-state-remote-mas") %>>
<a href="/docs/state/remote/mas.html">mas</a>
</li>
<li<%= sidebar_current("docs-state-remote-atlas") %>>
<a href="/docs/state/remote/atlas.html">atlas</a>
</li>