Merge pull request #1723 from apparentlymart/s3remotestate

S3 Remote State Backend

state/remote/remote.go

@@ -39,6 +39,7 @@ var BuiltinClients = map[string]Factory{
 	"atlas":  atlasFactory,
 	"consul": consulFactory,
 	"http":   httpFactory,
+	"s3":     s3Factory,
 
 	// This is used for development purposes only.
 	"_local": fileFactory,

state/remote/s3.go

@@ -0,0 +1,125 @@
+package remote
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/awslabs/aws-sdk-go/aws"
+	"github.com/awslabs/aws-sdk-go/service/s3"
+)
+
+func s3Factory(conf map[string]string) (Client, error) {
+	bucketName, ok := conf["bucket"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'bucket' configuration")
+	}
+
+	keyName, ok := conf["key"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'key' configuration")
+	}
+
+	regionName, ok := conf["region"]
+	if !ok {
+		regionName = os.Getenv("AWS_DEFAULT_REGION")
+		if regionName == "" {
+			return nil, fmt.Errorf("missing 'region' configuration or AWS_DEFAULT_REGION environment variable")
+		}
+	}
+
+	accessKeyId := conf["access_key"]
+	secretAccessKey := conf["secret_key"]
+
+	credentialsProvider := aws.DetectCreds(accessKeyId, secretAccessKey, "")
+
+	// Make sure we got some sort of working credentials.
+	_, err := credentialsProvider.Credentials()
+	if err != nil {
+		return nil, fmt.Errorf("Unable to determine AWS credentials. Set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.\n(error was: %s)", err)
+	}
+
+	awsConfig := &aws.Config{
+		Credentials: credentialsProvider,
+		Region:      regionName,
+	}
+	nativeClient := s3.New(awsConfig)
+
+	return &S3Client{
+		nativeClient: nativeClient,
+		bucketName:   bucketName,
+		keyName:      keyName,
+	}, nil
+}
+
+type S3Client struct {
+	nativeClient *s3.S3
+	bucketName   string
+	keyName      string
+}
+
+func (c *S3Client) Get() (*Payload, error) {
+	output, err := c.nativeClient.GetObject(&s3.GetObjectInput{
+		Bucket: &c.bucketName,
+		Key:    &c.keyName,
+	})
+
+	if err != nil {
+		if awserr := aws.Error(err); awserr != nil {
+			if awserr.Code == "NoSuchKey" {
+				return nil, nil
+			} else {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	defer output.Body.Close()
+
+	buf := bytes.NewBuffer(nil)
+	if _, err := io.Copy(buf, output.Body); err != nil {
+		return nil, fmt.Errorf("Failed to read remote state: %s", err)
+	}
+
+	payload := &Payload{
+		Data: buf.Bytes(),
+	}
+
+	// If there was no data, then return nil
+	if len(payload.Data) == 0 {
+		return nil, nil
+	}
+
+	return payload, nil
+}
+
+func (c *S3Client) Put(data []byte) error {
+	contentType := "application/octet-stream"
+	contentLength := int64(len(data))
+
+	_, err := c.nativeClient.PutObject(&s3.PutObjectInput{
+		ContentType:   &contentType,
+		ContentLength: &contentLength,
+		Body:          bytes.NewReader(data),
+		Bucket:        &c.bucketName,
+		Key:           &c.keyName,
+	})
+
+	if err == nil {
+		return nil
+	} else {
+		return fmt.Errorf("Failed to upload state: %v", err)
+	}
+}
+
+func (c *S3Client) Delete() error {
+	_, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: &c.bucketName,
+		Key:    &c.keyName,
+	})
+
+	return err
+}

state/remote/s3_test.go

@@ -0,0 +1,120 @@
+package remote
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/awslabs/aws-sdk-go/service/s3"
+)
+
+func TestS3Client_impl(t *testing.T) {
+	var _ Client = new(S3Client)
+}
+
+func TestS3Factory(t *testing.T) {
+	// This test just instantiates the client. Shouldn't make any actual
+	// requests nor incur any costs.
+
+	config := make(map[string]string)
+
+	// Empty config is an error
+	_, err := s3Factory(config)
+	if err == nil {
+		t.Fatalf("Empty config should be error")
+	}
+
+	config["region"] = "us-west-1"
+	config["bucket"] = "foo"
+	config["key"] = "bar"
+	// For this test we'll provide the credentials as config. The
+	// acceptance tests implicitly test passing credentials as
+	// environment variables.
+	config["access_key"] = "bazkey"
+	config["secret_key"] = "bazsecret"
+
+	client, err := s3Factory(config)
+	if err != nil {
+		t.Fatalf("Error for valid config")
+	}
+
+	s3Client := client.(*S3Client)
+
+	if s3Client.nativeClient.Config.Region != "us-west-1" {
+		t.Fatalf("Incorrect region was populated")
+	}
+	if s3Client.bucketName != "foo" {
+		t.Fatalf("Incorrect bucketName was populated")
+	}
+	if s3Client.keyName != "bar" {
+		t.Fatalf("Incorrect keyName was populated")
+	}
+
+	credentials, err := s3Client.nativeClient.Config.Credentials.Credentials()
+	if err != nil {
+		t.Fatalf("Error when requesting credentials")
+	}
+	if credentials.AccessKeyID != "bazkey" {
+		t.Fatalf("Incorrect Access Key Id was populated")
+	}
+	if credentials.SecretAccessKey != "bazsecret" {
+		t.Fatalf("Incorrect Secret Access Key was populated")
+	}
+}
+
+func TestS3Client(t *testing.T) {
+	// This test creates a bucket in S3 and populates it.
+	// It may incur costs, so it will only run if AWS credential environment
+	// variables are present.
+
+	accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID")
+	if accessKeyId == "" {
+		t.Skipf("skipping; AWS_ACCESS_KEY_ID must be set")
+	}
+
+	regionName := os.Getenv("AWS_DEFAULT_REGION")
+	if regionName == "" {
+		regionName = "us-west-2"
+	}
+
+	bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
+	keyName := "testState"
+
+	config := make(map[string]string)
+	config["region"] = regionName
+	config["bucket"] = bucketName
+	config["key"] = keyName
+
+	client, err := s3Factory(config)
+	if err != nil {
+		t.Fatalf("Error for valid config")
+	}
+
+	s3Client := client.(*S3Client)
+	nativeClient := s3Client.nativeClient
+
+	createBucketReq := &s3.CreateBucketInput{
+		Bucket: &bucketName,
+	}
+
+	// Be clear about what we're doing in case the user needs to clean
+	// this up later.
+	t.Logf("Creating S3 bucket %s in %s", bucketName, regionName)
+	_, err = nativeClient.CreateBucket(createBucketReq)
+	if err != nil {
+		t.Skipf("Failed to create test S3 bucket, so skipping")
+	}
+	defer func () {
+		deleteBucketReq := &s3.DeleteBucketInput{
+			Bucket: &bucketName,
+		}
+
+		_, err := nativeClient.DeleteBucket(deleteBucketReq)
+		if err != nil {
+			t.Logf("WARNING: Failed to delete the test S3 bucket. It has been left in your AWS account and may incur storage charges. (error was %s)", err)
+		}
+	}()
+
+	testClient(t, client)
+}

website/source/docs/commands/remote-config.html.markdown

@@ -50,6 +50,14 @@ The following backends are supported:
   variables can optionally be provided. Address is assumed to be the
   local agent if not provided.
 
+* S3 - Stores the state as a given key in a given bucket on Amazon S3.
+  Requires the `bucket` and `key` variables. Supports and honors the standard
+  AWS environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`
+  and `AWS_DEFAULT_REGION`. These can optionally be provided as parameters
+  in the `aws_access_key`, `aws_secret_key` and `region` variables
+  respectively, but passing credentials this way is not recommended since they
+  will be included in cleartext inside the persisted state.
+
 * HTTP - Stores the state using a simple REST client. State will be fetched
   via GET, updated via POST, and purged with DELETE. Requires the `address` variable.