remote state: Add GCS provider for remote state

command/remote_config.go

@@ -348,7 +348,7 @@ Usage: terraform remote config [options]
 Options:
 
   -backend=Atlas         Specifies the type of remote backend. Must be one
-                         of Atlas, Consul, Etcd, HTTP, S3, or Swift. Defaults
+                         of Atlas, Consul, Etcd, GCS, HTTP, S3, or Swift. Defaults
                          to Atlas.
 
   -backend-config="k=v"  Specifies configuration for the remote storage

state/remote/gcs.go

@@ -0,0 +1,175 @@
+package remote
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/hashicorp/terraform/helper/pathorcontents"
+	"github.com/hashicorp/terraform/terraform"
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"golang.org/x/oauth2/jwt"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/storage/v1"
+)
+
+// accountFile represents the structure of the credentials JSON
+type accountFile struct {
+	PrivateKeyId string `json:"private_key_id"`
+	PrivateKey   string `json:"private_key"`
+	ClientEmail  string `json:"client_email"`
+	ClientId     string `json:"client_id"`
+}
+
+func parseJSON(result interface{}, contents string) error {
+	r := strings.NewReader(contents)
+	dec := json.NewDecoder(r)
+
+	return dec.Decode(result)
+}
+
+type GCSClient struct {
+	bucket        string
+	path          string
+	clientStorage *storage.Service
+	context       context.Context
+}
+
+func gcsFactory(conf map[string]string) (Client, error) {
+	var account accountFile
+	var client *http.Client
+	clientScopes := []string{
+		"https://www.googleapis.com/auth/devstorage.full_control",
+	}
+
+	bucketName, ok := conf["bucket"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'bucket' configuration")
+	}
+
+	pathName, ok := conf["path"]
+	if !ok {
+		return nil, fmt.Errorf("missing 'path' configuration")
+	}
+
+	credentials, ok := conf["credentials"]
+	if !ok {
+		credentials = os.Getenv("GOOGLE_CREDENTIALS")
+	}
+
+	if credentials != "" {
+		contents, _, err := pathorcontents.Read(credentials)
+		if err != nil {
+			return nil, fmt.Errorf("Error loading credentials: %s", err)
+		}
+
+		// Assume account_file is a JSON string
+		if err := parseJSON(&account, contents); err != nil {
+			return nil, fmt.Errorf("Error parsing credentials '%s': %s", contents, err)
+		}
+
+		// Get the token for use in our requests
+		log.Printf("[INFO] Requesting Google token...")
+		log.Printf("[INFO]   -- Email: %s", account.ClientEmail)
+		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
+		log.Printf("[INFO]   -- Private Key Length: %d", len(account.PrivateKey))
+
+		conf := jwt.Config{
+			Email:      account.ClientEmail,
+			PrivateKey: []byte(account.PrivateKey),
+			Scopes:     clientScopes,
+			TokenURL:   "https://accounts.google.com/o/oauth2/token",
+		}
+
+		client = conf.Client(oauth2.NoContext)
+
+	} else {
+		log.Printf("[INFO] Authenticating using DefaultClient")
+		err := error(nil)
+		client, err = google.DefaultClient(oauth2.NoContext, clientScopes...)
+		if err != nil {
+			return nil, err
+		}
+	}
+	versionString := terraform.Version
+	userAgent := fmt.Sprintf(
+		"(%s %s) Terraform/%s", runtime.GOOS, runtime.GOARCH, versionString)
+
+	log.Printf("[INFO] Instantiating Google Storage Client...")
+	clientStorage, err := storage.New(client)
+	if err != nil {
+		return nil, err
+	}
+	clientStorage.UserAgent = userAgent
+
+	return &GCSClient{
+		clientStorage: clientStorage,
+		bucket:        bucketName,
+		path:          pathName,
+	}, nil
+
+}
+
+func (c *GCSClient) Get() (*Payload, error) {
+	// Read the object from bucket.
+	log.Printf("[INFO] Reading %s/%s", c.bucket, c.path)
+
+	resp, err := c.clientStorage.Objects.Get(c.bucket, c.path).Download()
+	if err != nil {
+		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
+			log.Printf("[INFO] %s/%s not found", c.bucket, c.path)
+
+			return nil, nil
+		}
+
+		return nil, fmt.Errorf("[WARN] Error retrieving object %s/%s: %s", c.bucket, c.path, err)
+	}
+	defer resp.Body.Close()
+
+	var buf []byte
+	w := bytes.NewBuffer(buf)
+	n, err := io.Copy(w, resp.Body)
+	if err != nil {
+		log.Fatalf("[WARN] error buffering %q: %v", c.path, err)
+	}
+	log.Printf("[INFO] Downloaded %d bytes", n)
+
+	payload := &Payload{
+		Data: w.Bytes(),
+	}
+
+	// If there was no data, then return nil
+	if len(payload.Data) == 0 {
+		return nil, nil
+	}
+
+	return payload, nil
+}
+
+func (c *GCSClient) Put(data []byte) error {
+	log.Printf("[INFO] Writing %s/%s", c.bucket, c.path)
+
+	r := bytes.NewReader(data)
+	_, err := c.clientStorage.Objects.Insert(c.bucket, &storage.Object{Name: c.path}).Media(r).Do()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *GCSClient) Delete() error {
+	log.Printf("[INFO] Deleting %s/%s", c.bucket, c.path)
+
+	err := c.clientStorage.Objects.Delete(c.bucket, c.path).Do()
+	return err
+
+}

state/remote/gcs_test.go

@@ -0,0 +1,69 @@
+package remote
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	storage "google.golang.org/api/storage/v1"
+)
+
+func TestGCSClient_impl(t *testing.T) {
+	var _ Client = new(GCSClient)
+}
+
+func TestGCSClient(t *testing.T) {
+	// This test creates a bucket in GCS and populates it.
+	// It may incur costs, so it will only run if GCS credential environment
+	// variables are present.
+
+	projectID := os.Getenv("GOOGLE_PROJECT")
+	if projectID == "" {
+		t.Skipf("skipping; GOOGLE_PROJECT must be set")
+	}
+
+	bucketName := fmt.Sprintf("terraform-remote-gcs-test-%x", time.Now().Unix())
+	keyName := "testState"
+	testData := []byte(`testing data`)
+
+	config := make(map[string]string)
+	config["bucket"] = bucketName
+	config["path"] = keyName
+
+	client, err := gcsFactory(config)
+	if err != nil {
+		t.Fatalf("Error for valid config: %v", err)
+	}
+
+	gcsClient := client.(*GCSClient)
+	nativeClient := gcsClient.clientStorage
+
+	// Be clear about what we're doing in case the user needs to clean
+	// this up later.
+	if _, err := nativeClient.Buckets.Get(bucketName).Do(); err == nil {
+		fmt.Printf("Bucket %s already exists - skipping buckets.insert call.", bucketName)
+	} else {
+		// Create a bucket.
+		if res, err := nativeClient.Buckets.Insert(projectID, &storage.Bucket{Name: bucketName}).Do(); err == nil {
+			fmt.Printf("Created bucket %v at location %v\n\n", res.Name, res.SelfLink)
+		} else {
+			t.Skipf("Failed to create test GCS bucket, so skipping")
+		}
+	}
+
+	// Ensure we can perform a PUT request with the encryption header
+	err = gcsClient.Put(testData)
+	if err != nil {
+		t.Logf("WARNING: Failed to send test data to GCS bucket. (error was %s)", err)
+	}
+
+	defer func() {
+		// Delete the test bucket in the project
+		if err := gcsClient.clientStorage.Buckets.Delete(bucketName).Do(); err != nil {
+			t.Logf("WARNING: Failed to delete the test GCS bucket. It has been left in your GCE account and may incur storage charges. (error was %s)", err)
+		}
+	}()
+
+	testClient(t, client)
+}

state/remote/remote.go

@@ -39,6 +39,7 @@ var BuiltinClients = map[string]Factory{
 	"atlas":       atlasFactory,
 	"consul":      consulFactory,
 	"etcd":        etcdFactory,
+	"gcs":         gcsFactory,
 	"http":        httpFactory,
 	"s3":          s3Factory,
 	"swift":       swiftFactory,

website/source/docs/state/remote/gcs.html.md

@@ -0,0 +1,55 @@
+---
+layout: "remotestate"
+page_title: "Remote State Backend: gcs"
+sidebar_current: "docs-state-remote-gcs"
+description: |-
+  Terraform can store the state remotely, making it easier to version and work with in a team.
+---
+
+# gcs
+
+Stores the state as a given key in a given bucket on [Google Cloud Storage](https://cloud.google.com/storage/).
+
+-> **Note:** Passing credentials directly via config options will
+make them included in cleartext inside the persisted state.
+Use of environment variables or config file is recommended.
+
+## Example Usage
+
+```
+terraform remote config \
+	-backend=gcs \
+	-backend-config="bucket=terraform-state-prod" \
+	-backend-config="path=network/terraform.tfstate" \
+	-backend-config="project=goopro"
+```
+
+## Example Referencing
+
+```hcl
+# setup remote state data source
+data "terraform_remote_state" "foo" {
+	backend = "gcs"
+	config {
+		bucket = "terraform-state-prod"
+		path = "network/terraform.tfstate"
+		project = "goopro"
+	}
+}
+
+# read value from data source
+resource "template_file" "bar" {
+  template = "${greeting}"
+
+  vars {
+    greeting = "${data.terraform_remote_state.foo.output.greeting}"
+  }
+}
+```
+
+## Configuration variables
+
+The following configuration options are supported:
+
+ * `bucket` - (Required) The name of the GCS bucket
+ * `path` - (Required) The path where to place/look for state file inside the bucket

website/source/layouts/remotestate.erb

@@ -25,6 +25,9 @@
 						<li<%= sidebar_current("docs-state-remote-etcd") %>>
 							<a href="/docs/state/remote/etcd.html">etcd</a>
 						</li>
+						<li<%= sidebar_current("docs-state-remote-gcs") %>>
+							<a href="/docs/state/remote/gcs.html">gcs</a>
+						</li>
 						<li<%= sidebar_current("docs-state-remote-http") %>>
 							<a href="/docs/state/remote/http.html">http</a>
 						</li>