Implemented GCloud backend supporting remote locking and multiple workspaces.
This commit is contained in:
Piotrek Bzdyl
James Bardin
parent
633d428c15
commit
5854373018
|
|
@ -14,6 +14,7 @@ import (
|
|||
backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure"
|
||||
backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul"
|
||||
backendetcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3"
|
||||
backendGCloud "github.com/hashicorp/terraform/backend/remote-state/gcloud"
|
||||
backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem"
|
||||
backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3"
|
||||
backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift"
|
||||
|
|
@ -47,6 +48,7 @@ func init() {
|
|||
`Warning: "azure" name is deprecated, please use "azurerm"`),
|
||||
"azurerm": func() backend.Backend { return backendAzure.New() },
|
||||
"etcdv3": func() backend.Backend { return backendetcdv3.New() },
|
||||
"gcloud": func() backend.Backend { return backendGCloud.New() },
|
||||
}
|
||||
|
||||
// Add the legacy remote backends that haven't yet been convertd to
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
package gcloud
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/storage"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/hashicorp/terraform/backend"
|
||||
"github.com/hashicorp/terraform/helper/pathorcontents"
|
||||
"github.com/hashicorp/terraform/helper/schema"
|
||||
googleContext "golang.org/x/net/context"
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
func New() backend.Backend {
|
||||
s := &schema.Backend{
|
||||
Schema: map[string]*schema.Schema{
|
||||
"bucket": {
|
||||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
Description: "The name of the Google Cloud Storage bucket",
|
||||
},
|
||||
|
||||
"state_dir": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
Description: "The directory where state files will be saved inside the bucket",
|
||||
},
|
||||
|
||||
"credentials": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
Description: "Google Cloud JSON Account Key",
|
||||
Default: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := &Backend{Backend: s}
|
||||
result.Backend.ConfigureFunc = result.configure
|
||||
return result
|
||||
}
|
||||
|
||||
type Backend struct {
|
||||
*schema.Backend
|
||||
|
||||
storageClient *storage.Client
|
||||
storageContext googleContext.Context
|
||||
|
||||
bucketName string
|
||||
stateDir string
|
||||
}
|
||||
|
||||
func (b *Backend) configure(ctx context.Context) error {
|
||||
if b.storageClient != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
storageOAuth2Scope := "https://www.googleapis.com/auth/devstorage.read_write"
|
||||
|
||||
data := schema.FromContextBackendConfig(ctx)
|
||||
|
||||
b.bucketName = data.Get("bucket").(string)
|
||||
b.stateDir = data.Get("state_dir").(string)
|
||||
b.storageContext = googleContext.Background()
|
||||
|
||||
credentials := data.Get("credentials").(string)
|
||||
|
||||
var tokenSource oauth2.TokenSource
|
||||
|
||||
if credentials != "" {
|
||||
credentialsJson, _, err := pathorcontents.Read(data.Get("credentials").(string))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error loading credentials: %v", err)
|
||||
}
|
||||
|
||||
jwtConfig, err := google.JWTConfigFromJSON([]byte(credentialsJson), storageOAuth2Scope)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get Google OAuth2 token: %v", err)
|
||||
}
|
||||
|
||||
tokenSource = jwtConfig.TokenSource(b.storageContext)
|
||||
} else {
|
||||
defaultTokenSource, err := google.DefaultTokenSource(b.storageContext, storageOAuth2Scope)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get Google Application Default Credentials: %v", err)
|
||||
}
|
||||
|
||||
tokenSource = defaultTokenSource
|
||||
}
|
||||
|
||||
client, err := storage.NewClient(b.storageContext, option.WithTokenSource(tokenSource))
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create Google Storage client: %v", err)
|
||||
}
|
||||
|
||||
b.storageClient = client
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
package gcloud
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/storage"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/hashicorp/terraform/backend"
|
||||
"github.com/hashicorp/terraform/state"
|
||||
"github.com/hashicorp/terraform/state/remote"
|
||||
"github.com/hashicorp/terraform/terraform"
|
||||
"google.golang.org/api/iterator"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (b *Backend) States() ([]string, error) {
|
||||
workspaces := []string{backend.DefaultStateName}
|
||||
var stateRegex *regexp.Regexp
|
||||
var err error
|
||||
if b.stateDir == "" {
|
||||
stateRegex = regexp.MustCompile(`^(.+)\.tfstate$`)
|
||||
} else {
|
||||
stateRegex, err = regexp.Compile(fmt.Sprintf("^%v/(.+)\\.tfstate$", regexp.QuoteMeta(b.stateDir)))
|
||||
if err != nil {
|
||||
return []string{}, fmt.Errorf("Failed to compile regex for querying states: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
bucket := b.storageClient.Bucket(b.bucketName)
|
||||
query := &storage.Query{
|
||||
Prefix: b.stateDir,
|
||||
}
|
||||
|
||||
files := bucket.Objects(b.storageContext, query)
|
||||
for {
|
||||
attrs, err := files.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return []string{}, fmt.Errorf("Failed to query remote states: %v", err)
|
||||
}
|
||||
|
||||
matches := stateRegex.FindStringSubmatch(attrs.Name)
|
||||
if len(matches) == 2 && matches[1] != backend.DefaultStateName {
|
||||
workspaces = append(workspaces, matches[1])
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(workspaces[1:])
|
||||
return workspaces, nil
|
||||
}
|
||||
|
||||
func (b *Backend) DeleteState(name string) error {
|
||||
if name == backend.DefaultStateName || name == "" {
|
||||
return fmt.Errorf("Can't delete default state")
|
||||
}
|
||||
|
||||
client, err := b.remoteClient(name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create Google Storage client: %v", err)
|
||||
}
|
||||
|
||||
err = client.Delete()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to delete state file %v: %v", client.stateFileUrl(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// get a remote client configured for this state
|
||||
func (b *Backend) remoteClient(name string) (*RemoteClient, error) {
|
||||
if name == "" {
|
||||
return nil, errors.New("Missing state name")
|
||||
}
|
||||
|
||||
client := &RemoteClient{
|
||||
storageContext: b.storageContext,
|
||||
storageClient: b.storageClient,
|
||||
bucketName: b.bucketName,
|
||||
stateFilePath: b.stateFileName(name),
|
||||
lockFilePath: b.lockFileName(name),
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (b *Backend) State(name string) (state.State, error) {
|
||||
client, err := b.remoteClient(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to create Google Storage client: %v", err)
|
||||
}
|
||||
|
||||
stateMgr := &remote.State{Client: client}
|
||||
lockInfo := state.NewLockInfo()
|
||||
lockInfo.Operation = "init"
|
||||
lockId, err := stateMgr.Lock(lockInfo)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to lock state in Google Cloud Storage: %v", err)
|
||||
}
|
||||
|
||||
// Local helper function so we can call it multiple places
|
||||
lockUnlock := func(parent error) error {
|
||||
if err := stateMgr.Unlock(lockId); err != nil {
|
||||
return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, client.lockFileUrl(), err)
|
||||
}
|
||||
|
||||
return parent
|
||||
}
|
||||
|
||||
// Grab the value
|
||||
if err := stateMgr.RefreshState(); err != nil {
|
||||
err = lockUnlock(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If we have no state, we have to create an empty state
|
||||
if v := stateMgr.State(); v == nil {
|
||||
if err := stateMgr.WriteState(terraform.NewState()); err != nil {
|
||||
err = lockUnlock(err)
|
||||
return nil, err
|
||||
}
|
||||
if err := stateMgr.PersistState(); err != nil {
|
||||
err = lockUnlock(err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Unlock, the state should now be initialized
|
||||
if err := lockUnlock(nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stateMgr, nil
|
||||
}
|
||||
|
||||
func (b *Backend) stateFileName(stateName string) string {
|
||||
if b.stateDir == "" {
|
||||
return fmt.Sprintf("%v.tfstate", stateName)
|
||||
} else {
|
||||
return fmt.Sprintf("%v/%v.tfstate", b.stateDir, stateName)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Backend) lockFileName(stateName string) string {
|
||||
if b.stateDir == "" {
|
||||
return fmt.Sprintf("%v.tflock", stateName)
|
||||
} else {
|
||||
return fmt.Sprintf("%v/%v.tflock", b.stateDir, stateName)
|
||||
}
|
||||
}
|
||||
|
||||
const errStateUnlock = `
|
||||
Error unlocking Google Cloud Storage state.
|
||||
|
||||
Lock ID: %v
|
||||
Lock file URL: %v
|
||||
Error: %v
|
||||
|
||||
You may have to force-unlock this state in order to use it again.
|
||||
The GCloud backend acquires a lock during initialization to ensure
|
||||
the initial state file is created.
|
||||
`
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
package gcloud
|
||||
|
||||
import (
|
||||
"cloud.google.com/go/storage"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
uuid "github.com/hashicorp/go-uuid"
|
||||
"github.com/hashicorp/terraform/state"
|
||||
"github.com/hashicorp/terraform/state/remote"
|
||||
"golang.org/x/net/context"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type RemoteClient struct {
|
||||
storageContext context.Context
|
||||
storageClient *storage.Client
|
||||
bucketName string
|
||||
stateFilePath string
|
||||
lockFilePath string
|
||||
}
|
||||
|
||||
func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
|
||||
bucket := c.storageClient.Bucket(c.bucketName)
|
||||
stateFile := bucket.Object(c.stateFilePath)
|
||||
stateFileUrl := c.stateFileUrl()
|
||||
|
||||
stateFileReader, err := stateFile.NewReader(c.storageContext)
|
||||
if err != nil {
|
||||
if err == storage.ErrObjectNotExist {
|
||||
return nil, nil
|
||||
} else {
|
||||
return nil, fmt.Errorf("Failed to open state file at %v: %v", stateFileUrl, err)
|
||||
}
|
||||
}
|
||||
defer stateFileReader.Close()
|
||||
|
||||
stateFileContents, err := ioutil.ReadAll(stateFileReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to read state file from %v: %v", stateFileUrl, err)
|
||||
}
|
||||
|
||||
stateFileAttrs, err := stateFile.Attrs(c.storageContext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to read state file attrs from %v: %v", stateFileUrl, err)
|
||||
}
|
||||
|
||||
result := &remote.Payload{
|
||||
Data: stateFileContents,
|
||||
MD5: stateFileAttrs.MD5,
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *RemoteClient) Put(data []byte) error {
|
||||
bucket := c.storageClient.Bucket(c.bucketName)
|
||||
stateFile := bucket.Object(c.stateFilePath)
|
||||
|
||||
stateFileWriter := stateFile.NewWriter(c.storageContext)
|
||||
|
||||
stateFileWriter.Write(data)
|
||||
err := stateFileWriter.Close()
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to upload state to %v: %v", c.stateFileUrl(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *RemoteClient) Delete() error {
|
||||
bucket := c.storageClient.Bucket(c.bucketName)
|
||||
stateFile := bucket.Object(c.stateFilePath)
|
||||
|
||||
err := stateFile.Delete(c.storageContext)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to delete state file %v: %v", c.stateFileUrl(), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
|
||||
if info.ID == "" {
|
||||
lockID, err := uuid.GenerateUUID()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
info.ID = lockID
|
||||
}
|
||||
|
||||
info.Path = c.lockFileUrl()
|
||||
|
||||
infoJson, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
bucket := c.storageClient.Bucket(c.bucketName)
|
||||
lockFile := bucket.Object(c.lockFilePath)
|
||||
|
||||
writer := lockFile.If(storage.Conditions{DoesNotExist: true}).NewWriter(c.storageContext)
|
||||
writer.Write(infoJson)
|
||||
if err := writer.Close(); err != nil {
|
||||
return "", fmt.Errorf("Error while saving lock file (%v): %v", info.Path, err)
|
||||
}
|
||||
|
||||
return info.ID, nil
|
||||
}
|
||||
|
||||
func (c *RemoteClient) Unlock(id string) error {
|
||||
lockErr := &state.LockError{}
|
||||
|
||||
bucket := c.storageClient.Bucket(c.bucketName)
|
||||
lockFile := bucket.Object(c.lockFilePath)
|
||||
lockFileUrl := c.lockFileUrl()
|
||||
|
||||
lockFileReader, err := lockFile.NewReader(c.storageContext)
|
||||
if err != nil {
|
||||
lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err)
|
||||
return lockErr
|
||||
}
|
||||
defer lockFileReader.Close()
|
||||
|
||||
lockFileContents, err := ioutil.ReadAll(lockFileReader)
|
||||
if err != nil {
|
||||
lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err)
|
||||
return lockErr
|
||||
}
|
||||
|
||||
lockInfo := &state.LockInfo{}
|
||||
err = json.Unmarshal(lockFileContents, lockInfo)
|
||||
if err != nil {
|
||||
lockErr.Err = fmt.Errorf("Failed to unmarshal lock info (%v): %v", lockFileUrl, err)
|
||||
return lockErr
|
||||
}
|
||||
|
||||
lockErr.Info = lockInfo
|
||||
|
||||
if lockInfo.ID != id {
|
||||
lockErr.Err = fmt.Errorf("Lock id %q does not match existing lock", id)
|
||||
return lockErr
|
||||
}
|
||||
|
||||
lockFileAttrs, err := lockFile.Attrs(c.storageContext)
|
||||
if err != nil {
|
||||
lockErr.Err = fmt.Errorf("Failed to fetch lock file attrs (%v): %v", lockFileUrl, err)
|
||||
return lockErr
|
||||
}
|
||||
|
||||
err = lockFile.If(storage.Conditions{GenerationMatch: lockFileAttrs.Generation}).Delete(c.storageContext)
|
||||
if err != nil {
|
||||
lockErr.Err = fmt.Errorf("Failed to delete lock file (%v): %v", lockFileUrl, err)
|
||||
return lockErr
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *RemoteClient) stateFileUrl() string {
|
||||
return fmt.Sprintf("gs://%v/%v", c.bucketName, c.stateFilePath)
|
||||
}
|
||||
|
||||
func (c *RemoteClient) lockFileUrl() string {
|
||||
return fmt.Sprintf("gs://%v/%v", c.bucketName, c.lockFilePath)
|
||||
}
|
||||
Loading…
Reference in New Issue