diff --git a/command/remote_config.go b/command/remote_config.go index d7e73a5d8..afe6613db 100644 --- a/command/remote_config.go +++ b/command/remote_config.go @@ -348,7 +348,7 @@ Usage: terraform remote config [options] Options: -backend=Atlas Specifies the type of remote backend. Must be one - of Atlas, Consul, Etcd, HTTP, S3, or Swift. Defaults + of Atlas, Consul, Etcd, GCS, HTTP, S3, or Swift. Defaults to Atlas. -backend-config="k=v" Specifies configuration for the remote storage diff --git a/state/remote/gcs.go b/state/remote/gcs.go new file mode 100644 index 000000000..8ea682fef --- /dev/null +++ b/state/remote/gcs.go @@ -0,0 +1,175 @@ +package remote + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "runtime" + "strings" + + "github.com/hashicorp/terraform/helper/pathorcontents" + "github.com/hashicorp/terraform/terraform" + "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/googleapi" + "google.golang.org/api/storage/v1" +) + +// accountFile represents the structure of the credentials JSON +type accountFile struct { + PrivateKeyId string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientId string `json:"client_id"` +} + +func parseJSON(result interface{}, contents string) error { + r := strings.NewReader(contents) + dec := json.NewDecoder(r) + + return dec.Decode(result) +} + +type GCSClient struct { + bucket string + path string + clientStorage *storage.Service + context context.Context +} + +func gcsFactory(conf map[string]string) (Client, error) { + var account accountFile + var client *http.Client + clientScopes := []string{ + "https://www.googleapis.com/auth/devstorage.full_control", + } + + bucketName, ok := conf["bucket"] + if !ok { + return nil, fmt.Errorf("missing 'bucket' configuration") + } + + pathName, ok := conf["path"] + if !ok { + return nil, fmt.Errorf("missing 'path' configuration") + } + + credentials, ok := conf["credentials"] + if !ok { + credentials = os.Getenv("GOOGLE_CREDENTIALS") + } + + if credentials != "" { + contents, _, err := pathorcontents.Read(credentials) + if err != nil { + return nil, fmt.Errorf("Error loading credentials: %s", err) + } + + // Assume account_file is a JSON string + if err := parseJSON(&account, contents); err != nil { + return nil, fmt.Errorf("Error parsing credentials '%s': %s", contents, err) + } + + // Get the token for use in our requests + log.Printf("[INFO] Requesting Google token...") + log.Printf("[INFO] -- Email: %s", account.ClientEmail) + log.Printf("[INFO] -- Scopes: %s", clientScopes) + log.Printf("[INFO] -- Private Key Length: %d", len(account.PrivateKey)) + + conf := jwt.Config{ + Email: account.ClientEmail, + PrivateKey: []byte(account.PrivateKey), + Scopes: clientScopes, + TokenURL: "https://accounts.google.com/o/oauth2/token", + } + + client = conf.Client(oauth2.NoContext) + + } else { + log.Printf("[INFO] Authenticating using DefaultClient") + err := error(nil) + client, err = google.DefaultClient(oauth2.NoContext, clientScopes...) + if err != nil { + return nil, err + } + } + versionString := terraform.Version + userAgent := fmt.Sprintf( + "(%s %s) Terraform/%s", runtime.GOOS, runtime.GOARCH, versionString) + + log.Printf("[INFO] Instantiating Google Storage Client...") + clientStorage, err := storage.New(client) + if err != nil { + return nil, err + } + clientStorage.UserAgent = userAgent + + return &GCSClient{ + clientStorage: clientStorage, + bucket: bucketName, + path: pathName, + }, nil + +} + +func (c *GCSClient) Get() (*Payload, error) { + // Read the object from bucket. + log.Printf("[INFO] Reading %s/%s", c.bucket, c.path) + + resp, err := c.clientStorage.Objects.Get(c.bucket, c.path).Download() + if err != nil { + if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 { + log.Printf("[INFO] %s/%s not found", c.bucket, c.path) + + return nil, nil + } + + return nil, fmt.Errorf("[WARN] Error retrieving object %s/%s: %s", c.bucket, c.path, err) + } + defer resp.Body.Close() + + var buf []byte + w := bytes.NewBuffer(buf) + n, err := io.Copy(w, resp.Body) + if err != nil { + log.Fatalf("[WARN] error buffering %q: %v", c.path, err) + } + log.Printf("[INFO] Downloaded %d bytes", n) + + payload := &Payload{ + Data: w.Bytes(), + } + + // If there was no data, then return nil + if len(payload.Data) == 0 { + return nil, nil + } + + return payload, nil +} + +func (c *GCSClient) Put(data []byte) error { + log.Printf("[INFO] Writing %s/%s", c.bucket, c.path) + + r := bytes.NewReader(data) + _, err := c.clientStorage.Objects.Insert(c.bucket, &storage.Object{Name: c.path}).Media(r).Do() + if err != nil { + return err + } + + return nil +} + +func (c *GCSClient) Delete() error { + log.Printf("[INFO] Deleting %s/%s", c.bucket, c.path) + + err := c.clientStorage.Objects.Delete(c.bucket, c.path).Do() + return err + +} diff --git a/state/remote/gcs_test.go b/state/remote/gcs_test.go new file mode 100644 index 000000000..402093917 --- /dev/null +++ b/state/remote/gcs_test.go @@ -0,0 +1,69 @@ +package remote + +import ( + "fmt" + "os" + "testing" + "time" + + storage "google.golang.org/api/storage/v1" +) + +func TestGCSClient_impl(t *testing.T) { + var _ Client = new(GCSClient) +} + +func TestGCSClient(t *testing.T) { + // This test creates a bucket in GCS and populates it. + // It may incur costs, so it will only run if GCS credential environment + // variables are present. + + projectID := os.Getenv("GOOGLE_PROJECT") + if projectID == "" { + t.Skipf("skipping; GOOGLE_PROJECT must be set") + } + + bucketName := fmt.Sprintf("terraform-remote-gcs-test-%x", time.Now().Unix()) + keyName := "testState" + testData := []byte(`testing data`) + + config := make(map[string]string) + config["bucket"] = bucketName + config["path"] = keyName + + client, err := gcsFactory(config) + if err != nil { + t.Fatalf("Error for valid config: %v", err) + } + + gcsClient := client.(*GCSClient) + nativeClient := gcsClient.clientStorage + + // Be clear about what we're doing in case the user needs to clean + // this up later. + if _, err := nativeClient.Buckets.Get(bucketName).Do(); err == nil { + fmt.Printf("Bucket %s already exists - skipping buckets.insert call.", bucketName) + } else { + // Create a bucket. + if res, err := nativeClient.Buckets.Insert(projectID, &storage.Bucket{Name: bucketName}).Do(); err == nil { + fmt.Printf("Created bucket %v at location %v\n\n", res.Name, res.SelfLink) + } else { + t.Skipf("Failed to create test GCS bucket, so skipping") + } + } + + // Ensure we can perform a PUT request with the encryption header + err = gcsClient.Put(testData) + if err != nil { + t.Logf("WARNING: Failed to send test data to GCS bucket. (error was %s)", err) + } + + defer func() { + // Delete the test bucket in the project + if err := gcsClient.clientStorage.Buckets.Delete(bucketName).Do(); err != nil { + t.Logf("WARNING: Failed to delete the test GCS bucket. It has been left in your GCE account and may incur storage charges. (error was %s)", err) + } + }() + + testClient(t, client) +} diff --git a/state/remote/remote.go b/state/remote/remote.go index 4074c2c64..7abd40e1e 100644 --- a/state/remote/remote.go +++ b/state/remote/remote.go @@ -39,6 +39,7 @@ var BuiltinClients = map[string]Factory{ "atlas": atlasFactory, "consul": consulFactory, "etcd": etcdFactory, + "gcs": gcsFactory, "http": httpFactory, "s3": s3Factory, "swift": swiftFactory, diff --git a/website/source/docs/state/remote/gcs.html.md b/website/source/docs/state/remote/gcs.html.md new file mode 100644 index 000000000..d5a820664 --- /dev/null +++ b/website/source/docs/state/remote/gcs.html.md @@ -0,0 +1,55 @@ +--- +layout: "remotestate" +page_title: "Remote State Backend: gcs" +sidebar_current: "docs-state-remote-gcs" +description: |- + Terraform can store the state remotely, making it easier to version and work with in a team. +--- + +# gcs + +Stores the state as a given key in a given bucket on [Google Cloud Storage](https://cloud.google.com/storage/). + +-> **Note:** Passing credentials directly via config options will +make them included in cleartext inside the persisted state. +Use of environment variables or config file is recommended. + +## Example Usage + +``` +terraform remote config \ + -backend=gcs \ + -backend-config="bucket=terraform-state-prod" \ + -backend-config="path=network/terraform.tfstate" \ + -backend-config="project=goopro" +``` + +## Example Referencing + +```hcl +# setup remote state data source +data "terraform_remote_state" "foo" { + backend = "gcs" + config { + bucket = "terraform-state-prod" + path = "network/terraform.tfstate" + project = "goopro" + } +} + +# read value from data source +resource "template_file" "bar" { + template = "${greeting}" + + vars { + greeting = "${data.terraform_remote_state.foo.output.greeting}" + } +} +``` + +## Configuration variables + +The following configuration options are supported: + + * `bucket` - (Required) The name of the GCS bucket + * `path` - (Required) The path where to place/look for state file inside the bucket diff --git a/website/source/layouts/remotestate.erb b/website/source/layouts/remotestate.erb index 7a4c57af2..9dc90172d 100644 --- a/website/source/layouts/remotestate.erb +++ b/website/source/layouts/remotestate.erb @@ -25,6 +25,9 @@ > etcd + > + gcs + > http