From 7fb2d1b8de6cc0bf6b815910b7410f7d7307a02d Mon Sep 17 00:00:00 2001 From: Sander van Harmelen Date: Wed, 4 Jul 2018 17:24:49 +0200 Subject: [PATCH] Implement the Enterprise enhanced remote backend --- backend/backend.go | 23 +- backend/init/init.go | 39 +- backend/init/init_test.go | 110 +++++ backend/legacy/legacy.go | 4 +- backend/legacy/legacy_test.go | 4 +- backend/local/testing.go | 44 ++ backend/remote/backend.go | 453 ++++++++++++++++++ backend/remote/backend_mock.go | 384 +++++++++++++++ backend/remote/backend_plan.go | 206 ++++++++ backend/remote/backend_plan_test.go | 181 +++++++ backend/remote/backend_state.go | 103 ++++ backend/remote/backend_state_test.go | 16 + backend/remote/backend_test.go | 254 ++++++++++ backend/remote/cli.go | 13 + .../test-fixtures/plan-scaleout/main.tf | 10 + backend/remote/test-fixtures/plan/main.tf | 1 + backend/remote/test-fixtures/plan/output.log | 29 ++ backend/remote/testing.go | 128 +++++ backend/testing.go | 28 +- builtin/providers/terraform/provider_test.go | 4 + command/command.go | 4 - command/command_test.go | 4 + command/init.go | 9 +- command/meta_backend_migrate.go | 14 +- command/meta_backend_test.go | 106 ++++ .../.terraform/terraform.tfstate | 22 + .../local-state.tfstate | 6 + .../main.tf | 5 + .../env2/terraform.tfstate | 6 + .../.terraform/terraform.tfstate | 22 + .../main.tf | 5 + .../env2/terraform.tfstate | 6 + main.go | 14 +- state/remote/testing.go | 4 +- website/docs/backends/types/remote.html.md | 118 +++++ .../types/terraform-enterprise.html.md | 3 + website/layouts/backend-types.erb | 3 + 37 files changed, 2342 insertions(+), 43 deletions(-) create mode 100644 backend/init/init_test.go create mode 100644 backend/remote/backend.go create mode 100644 backend/remote/backend_mock.go create mode 100644 backend/remote/backend_plan.go create mode 100644 backend/remote/backend_plan_test.go create mode 100644 backend/remote/backend_state.go create mode 100644 backend/remote/backend_state_test.go create mode 100644 backend/remote/backend_test.go create mode 100644 backend/remote/cli.go create mode 100644 backend/remote/test-fixtures/plan-scaleout/main.tf create mode 100644 backend/remote/test-fixtures/plan/main.tf create mode 100644 backend/remote/test-fixtures/plan/output.log create mode 100644 backend/remote/testing.go create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-with-default/.terraform/terraform.tfstate create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-with-default/local-state.tfstate create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-with-default/main.tf create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-with-default/terraform.tfstate.d/env2/terraform.tfstate create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-without-default/.terraform/terraform.tfstate create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-without-default/main.tf create mode 100644 command/test-fixtures/backend-change-multi-to-no-default-without-default/terraform.tfstate.d/env2/terraform.tfstate create mode 100644 website/docs/backends/types/remote.html.md diff --git a/backend/backend.go b/backend/backend.go index dfeb80ef6..f10c27c9f 100644 --- a/backend/backend.go +++ b/backend/backend.go @@ -15,14 +15,29 @@ import ( "github.com/hashicorp/terraform/terraform" ) -// This is the name of the default, initial state that every backend -// must have. This state cannot be deleted. +// DefaultStateName is the name of the default, initial state that every +// backend must have. This state cannot be deleted. const DefaultStateName = "default" -// Error value to return when a named state operation isn't supported. // This must be returned rather than a custom error so that the Terraform // CLI can detect it and handle it appropriately. -var ErrNamedStatesNotSupported = errors.New("named states not supported") +var ( + // ErrNamedStatesNotSupported is returned when a named state operation + // isn't supported. + ErrNamedStatesNotSupported = errors.New("named states not supported") + + // ErrDefaultStateNotSupported is returned when an operation does not support + // using the default state, but requires a named state to be selected. + ErrDefaultStateNotSupported = errors.New("default state not supported\n\n" + + "You can create a new workspace wth the \"workspace new\" command") + + // ErrOperationNotSupported is returned when an unsupported operation + // is detected by the configured backend. + ErrOperationNotSupported = errors.New("operation not supported") +) + +// InitFn is used to initialize a new backend. +type InitFn func() Backend // Backend is the minimal interface that must be implemented to enable Terraform. type Backend interface { diff --git a/backend/init/init.go b/backend/init/init.go index 056827905..8fcd249c4 100644 --- a/backend/init/init.go +++ b/backend/init/init.go @@ -3,14 +3,17 @@ package init import ( + "os" "sync" "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/svchost/disco" "github.com/hashicorp/terraform/terraform" backendAtlas "github.com/hashicorp/terraform/backend/atlas" backendLegacy "github.com/hashicorp/terraform/backend/legacy" backendLocal "github.com/hashicorp/terraform/backend/local" + backendRemote "github.com/hashicorp/terraform/backend/remote" backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure" backendConsul "github.com/hashicorp/terraform/backend/remote-state/consul" backendEtcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3" @@ -32,17 +35,27 @@ import ( // complex structures and supporting that over the plugin system is currently // prohibitively difficult. For those wanting to implement a custom backend, // they can do so with recompilation. -var backends map[string]func() backend.Backend +var backends map[string]backend.InitFn var backendsLock sync.Mutex -func init() { - // Our hardcoded backends. We don't need to acquire a lock here - // since init() code is serial and can't spawn goroutines. - backends = map[string]func() backend.Backend{ +// Init initializes the backends map with all our hardcoded backends. +func Init(services *disco.Disco) { + backendsLock.Lock() + defer backendsLock.Unlock() + + backends = map[string]backend.InitFn{ + // Enhanced backends. "local": func() backend.Backend { return backendLocal.New() }, - "atlas": func() backend.Backend { return backendAtlas.New() }, - "azure": deprecateBackend(backendAzure.New(), - `Warning: "azure" name is deprecated, please use "azurerm"`), + "remote": func() backend.Backend { + b := backendRemote.New(services) + if os.Getenv("TF_FORCE_LOCAL_BACKEND") != "" { + return backendLocal.NewWithBackend(b) + } + return b + }, + + // Remote State backends. + "atlas": func() backend.Backend { return backendAtlas.New() }, "azurerm": func() backend.Backend { return backendAzure.New() }, "consul": func() backend.Backend { return backendConsul.New() }, "etcdv3": func() backend.Backend { return backendEtcdv3.New() }, @@ -51,6 +64,10 @@ func init() { "manta": func() backend.Backend { return backendManta.New() }, "s3": func() backend.Backend { return backendS3.New() }, "swift": func() backend.Backend { return backendSwift.New() }, + + // Deprecated backends. + "azure": deprecateBackend(backendAzure.New(), + `Warning: "azure" name is deprecated, please use "azurerm"`), } // Add the legacy remote backends that haven't yet been converted to @@ -60,7 +77,7 @@ func init() { // Backend returns the initialization factory for the given backend, or // nil if none exists. -func Backend(name string) func() backend.Backend { +func Backend(name string) backend.InitFn { backendsLock.Lock() defer backendsLock.Unlock() return backends[name] @@ -73,7 +90,7 @@ func Backend(name string) func() backend.Backend { // This method sets this backend globally and care should be taken to do // this only before Terraform is executing to prevent odd behavior of backends // changing mid-execution. -func Set(name string, f func() backend.Backend) { +func Set(name string, f backend.InitFn) { backendsLock.Lock() defer backendsLock.Unlock() @@ -101,7 +118,7 @@ func (b deprecatedBackendShim) Validate(c *terraform.ResourceConfig) ([]string, // DeprecateBackend can be used to wrap a backend to retrun a deprecation // warning during validation. -func deprecateBackend(b backend.Backend, message string) func() backend.Backend { +func deprecateBackend(b backend.Backend, message string) backend.InitFn { // Since a Backend wrapped by deprecatedBackendShim can no longer be // asserted as an Enhanced or Local backend, disallow those types here // entirely. If something other than a basic backend.Backend needs to be diff --git a/backend/init/init_test.go b/backend/init/init_test.go new file mode 100644 index 000000000..150b4c101 --- /dev/null +++ b/backend/init/init_test.go @@ -0,0 +1,110 @@ +package init + +import ( + "os" + "reflect" + "testing" + + backendLocal "github.com/hashicorp/terraform/backend/local" +) + +func TestInit_backend(t *testing.T) { + // Initialize the backends map + Init(nil) + + backends := []struct { + Name string + Type string + }{ + { + "local", + "*local.Local", + }, { + "remote", + "*remote.Remote", + }, { + "atlas", + "*atlas.Backend", + }, { + "azurerm", + "*azure.Backend", + }, { + "consul", + "*consul.Backend", + }, { + "etcdv3", + "*etcd.Backend", + }, { + "gcs", + "*gcs.Backend", + }, { + "inmem", + "*inmem.Backend", + }, { + "manta", + "*manta.Backend", + }, { + "s3", + "*s3.Backend", + }, { + "swift", + "*swift.Backend", + }, { + "azure", + "init.deprecatedBackendShim", + }, + } + + // Make sure we get the requested backend + for _, b := range backends { + f := Backend(b.Name) + bType := reflect.TypeOf(f()).String() + + if bType != b.Type { + t.Fatalf("expected backend %q to be %q, got: %q", b.Name, b.Type, bType) + } + } +} + +func TestInit_forceLocalBackend(t *testing.T) { + // Initialize the backends map + Init(nil) + + enhancedBackends := []struct { + Name string + Type string + }{ + { + "local", + "nil", + }, { + "remote", + "*remote.Remote", + }, + } + + // Set the TF_FORCE_LOCAL_BACKEND flag so all enhanced backends will + // return a local.Local backend with themselves as embedded backend. + if err := os.Setenv("TF_FORCE_LOCAL_BACKEND", "1"); err != nil { + t.Fatalf("error setting environment variable TF_FORCE_LOCAL_BACKEND: %v", err) + } + + // Make sure we always get the local backend. + for _, b := range enhancedBackends { + f := Backend(b.Name) + + local, ok := f().(*backendLocal.Local) + if !ok { + t.Fatalf("expected backend %q to be \"*local.Local\", got: %T", b.Name, f()) + } + + bType := "nil" + if local.Backend != nil { + bType = reflect.TypeOf(local.Backend).String() + } + + if bType != b.Type { + t.Fatalf("expected local.Backend to be %s, got: %s", b.Type, bType) + } + } +} diff --git a/backend/legacy/legacy.go b/backend/legacy/legacy.go index be3163bd5..6ed3e41d3 100644 --- a/backend/legacy/legacy.go +++ b/backend/legacy/legacy.go @@ -12,8 +12,8 @@ import ( // // If a type is already in the map, it will not be added. This will allow // us to slowly convert the legacy types to first-class backends. -func Init(m map[string]func() backend.Backend) { - for k, _ := range remote.BuiltinClients { +func Init(m map[string]backend.InitFn) { + for k := range remote.BuiltinClients { if _, ok := m[k]; !ok { // Copy the "k" value since the variable "k" is reused for // each key (address doesn't change). diff --git a/backend/legacy/legacy_test.go b/backend/legacy/legacy_test.go index 77d81bf1a..8a13c2577 100644 --- a/backend/legacy/legacy_test.go +++ b/backend/legacy/legacy_test.go @@ -8,7 +8,7 @@ import ( ) func TestInit(t *testing.T) { - m := make(map[string]func() backend.Backend) + m := make(map[string]backend.InitFn) Init(m) for k, _ := range remote.BuiltinClients { @@ -24,7 +24,7 @@ func TestInit(t *testing.T) { } func TestInit_ignoreExisting(t *testing.T) { - m := make(map[string]func() backend.Backend) + m := make(map[string]backend.InitFn) m["local"] = nil Init(m) diff --git a/backend/local/testing.go b/backend/local/testing.go index bd07fc886..6e8711860 100644 --- a/backend/local/testing.go +++ b/backend/local/testing.go @@ -99,6 +99,50 @@ func (b *TestLocalSingleState) DeleteState(string) error { return backend.ErrNamedStatesNotSupported } +// TestNewLocalNoDefault is a factory for creating a TestLocalNoDefaultState. +// This function matches the signature required for backend/init. +func TestNewLocalNoDefault() backend.Backend { + return &TestLocalNoDefaultState{Local: New()} +} + +// TestLocalNoDefaultState is a backend implementation that wraps +// Local and modifies it to support named states, but not the +// default state. It returns ErrDefaultStateNotSupported when the +// DefaultStateName is used. +type TestLocalNoDefaultState struct { + *Local +} + +func (b *TestLocalNoDefaultState) State(name string) (state.State, error) { + if name == backend.DefaultStateName { + return nil, backend.ErrDefaultStateNotSupported + } + return b.Local.State(name) +} + +func (b *TestLocalNoDefaultState) States() ([]string, error) { + states, err := b.Local.States() + if err != nil { + return nil, err + } + + filtered := states[:0] + for _, name := range states { + if name != backend.DefaultStateName { + filtered = append(filtered, name) + } + } + + return filtered, nil +} + +func (b *TestLocalNoDefaultState) DeleteState(name string) error { + if name == backend.DefaultStateName { + return backend.ErrDefaultStateNotSupported + } + return b.Local.DeleteState(name) +} + func testTempDir(t *testing.T) string { d, err := ioutil.TempDir("", "tf") if err != nil { diff --git a/backend/remote/backend.go b/backend/remote/backend.go new file mode 100644 index 000000000..30a990915 --- /dev/null +++ b/backend/remote/backend.go @@ -0,0 +1,453 @@ +package remote + +import ( + "context" + "fmt" + "log" + "net/url" + "sort" + "strings" + "sync" + + tfe "github.com/hashicorp/go-tfe" + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/helper/schema" + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" + "github.com/hashicorp/terraform/svchost" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/hashicorp/terraform/terraform" + "github.com/hashicorp/terraform/version" + "github.com/mitchellh/cli" + "github.com/mitchellh/colorstring" +) + +const ( + defaultHostname = "app.terraform.io" + serviceID = "tfe.v2" +) + +// Remote is an implementation of EnhancedBackend that performs all +// operations in a remote backend. +type Remote struct { + // CLI and Colorize control the CLI output. If CLI is nil then no CLI + // output will be done. If CLIColor is nil then no coloring will be done. + CLI cli.Ui + CLIColor *colorstring.Colorize + + // ContextOpts are the base context options to set when initializing a + // new Terraform context. Many of these will be overridden or merged by + // Operation. See Operation for more details. + ContextOpts *terraform.ContextOpts + + // client is the remote backend API client + client *tfe.Client + + // hostname of the remote backend server + hostname string + + // organization is the organization that contains the target workspaces + organization string + + // workspace is used to map the default workspace to a remote workspace + workspace string + + // prefix is used to filter down a set of workspaces that use a single + // configuration + prefix string + + // schema defines the configuration for the backend + schema *schema.Backend + + // services is used for service discovery + services *disco.Disco + + // opLock locks operations + opLock sync.Mutex +} + +// New creates a new initialized remote backend. +func New(services *disco.Disco) *Remote { + b := &Remote{ + services: services, + } + + b.schema = &schema.Backend{ + Schema: map[string]*schema.Schema{ + "hostname": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: schemaDescriptions["hostname"], + Default: defaultHostname, + }, + + "organization": &schema.Schema{ + Type: schema.TypeString, + Required: true, + Description: schemaDescriptions["organization"], + }, + + "token": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: schemaDescriptions["token"], + DefaultFunc: schema.EnvDefaultFunc("TFE_TOKEN", ""), + }, + + "workspaces": &schema.Schema{ + Type: schema.TypeSet, + Required: true, + Description: schemaDescriptions["workspaces"], + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: schemaDescriptions["name"], + }, + + "prefix": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Description: schemaDescriptions["prefix"], + }, + }, + }, + }, + }, + + ConfigureFunc: b.configure, + } + + return b +} + +func (b *Remote) configure(ctx context.Context) error { + d := schema.FromContextBackendConfig(ctx) + + // Get the hostname and organization. + b.hostname = d.Get("hostname").(string) + b.organization = d.Get("organization").(string) + + // Get the workspaces configuration. + workspaces := d.Get("workspaces").(*schema.Set) + if workspaces.Len() != 1 { + return fmt.Errorf("only one 'workspaces' block allowed") + } + + // After checking that we have exactly one workspace block, we can now get + // and assert that one workspace from the set. + workspace := workspaces.List()[0].(map[string]interface{}) + + // Get the default workspace name and prefix. + b.workspace = workspace["name"].(string) + b.prefix = workspace["prefix"].(string) + + // Make sure that we have either a workspace name or a prefix. + if b.workspace == "" && b.prefix == "" { + return fmt.Errorf("either workspace 'name' or 'prefix' is required") + } + + // Make sure that only one of workspace name or a prefix is configured. + if b.workspace != "" && b.prefix != "" { + return fmt.Errorf("only one of workspace 'name' or 'prefix' is allowed") + } + + // Discover the service URL for this host to confirm that it provides + // a remote backend API and to discover the required base path. + service, err := b.discover(b.hostname) + if err != nil { + return err + } + + // Retrieve the token for this host as configured in the credentials + // section of the CLI Config File. + token, err := b.token(b.hostname) + if err != nil { + return err + } + if token == "" { + token = d.Get("token").(string) + } + + cfg := &tfe.Config{ + Address: service.String(), + BasePath: service.Path, + Token: token, + } + + // Create the remote backend API client. + b.client, err = tfe.NewClient(cfg) + if err != nil { + return err + } + + return nil +} + +// discover the remote backend API service URL and token. +func (b *Remote) discover(hostname string) (*url.URL, error) { + host, err := svchost.ForComparison(hostname) + if err != nil { + return nil, err + } + service := b.services.DiscoverServiceURL(host, serviceID) + if service == nil { + return nil, fmt.Errorf("host %s does not provide a remote backend API", host) + } + return service, nil +} + +// token returns the token for this host as configured in the credentials +// section of the CLI Config File. If no token was configured, an empty +// string will be returned instead. +func (b *Remote) token(hostname string) (string, error) { + host, err := svchost.ForComparison(hostname) + if err != nil { + return "", err + } + creds, err := b.services.CredentialsForHost(host) + if err != nil { + log.Printf("[WARN] Failed to get credentials for %s: %s (ignoring)", host, err) + return "", nil + } + if creds != nil { + return creds.Token(), nil + } + return "", nil +} + +// Input is called to ask the user for input for completing the configuration. +func (b *Remote) Input(ui terraform.UIInput, c *terraform.ResourceConfig) (*terraform.ResourceConfig, error) { + return b.schema.Input(ui, c) +} + +// Validate is called once at the beginning with the raw configuration and +// can return a list of warnings and/or errors. +func (b *Remote) Validate(c *terraform.ResourceConfig) ([]string, []error) { + return b.schema.Validate(c) +} + +// Configure configures the backend itself with the configuration given. +func (b *Remote) Configure(c *terraform.ResourceConfig) error { + return b.schema.Configure(c) +} + +// State returns the latest state of the given remote workspace. The workspace +// will be created if it doesn't exist. +func (b *Remote) State(workspace string) (state.State, error) { + if b.workspace == "" && workspace == backend.DefaultStateName { + return nil, backend.ErrDefaultStateNotSupported + } + if b.prefix == "" && workspace != backend.DefaultStateName { + return nil, backend.ErrNamedStatesNotSupported + } + + workspaces, err := b.states() + if err != nil { + return nil, fmt.Errorf("Error retrieving workspaces: %v", err) + } + + exists := false + for _, name := range workspaces { + if workspace == name { + exists = true + break + } + } + + // Configure the remote workspace name. + if workspace == backend.DefaultStateName { + workspace = b.workspace + } else if b.prefix != "" && !strings.HasPrefix(workspace, b.prefix) { + workspace = b.prefix + workspace + } + + if !exists { + options := tfe.WorkspaceCreateOptions{ + Name: tfe.String(workspace), + TerraformVersion: tfe.String(version.Version), + } + _, err = b.client.Workspaces.Create(context.Background(), b.organization, options) + if err != nil { + return nil, fmt.Errorf("Error creating workspace %s: %v", workspace, err) + } + } + + client := &remoteClient{ + client: b.client, + organization: b.organization, + workspace: workspace, + } + + return &remote.State{Client: client}, nil +} + +// DeleteState removes the remote workspace if it exists. +func (b *Remote) DeleteState(workspace string) error { + if b.workspace == "" && workspace == backend.DefaultStateName { + return backend.ErrDefaultStateNotSupported + } + if b.prefix == "" && workspace != backend.DefaultStateName { + return backend.ErrNamedStatesNotSupported + } + + // Configure the remote workspace name. + if workspace == backend.DefaultStateName { + workspace = b.workspace + } else if b.prefix != "" && !strings.HasPrefix(workspace, b.prefix) { + workspace = b.prefix + workspace + } + + // Check if the configured organization exists. + _, err := b.client.Organizations.Read(context.Background(), b.organization) + if err != nil { + if err == tfe.ErrResourceNotFound { + return fmt.Errorf("organization %s does not exist", b.organization) + } + return err + } + + client := &remoteClient{ + client: b.client, + organization: b.organization, + workspace: workspace, + } + + return client.Delete() +} + +// States returns a filtered list of remote workspace names. +func (b *Remote) States() ([]string, error) { + if b.prefix == "" { + return nil, backend.ErrNamedStatesNotSupported + } + return b.states() +} + +func (b *Remote) states() ([]string, error) { + // Check if the configured organization exists. + _, err := b.client.Organizations.Read(context.Background(), b.organization) + if err != nil { + if err == tfe.ErrResourceNotFound { + return nil, fmt.Errorf("organization %s does not exist", b.organization) + } + return nil, err + } + + options := tfe.WorkspaceListOptions{} + ws, err := b.client.Workspaces.List(context.Background(), b.organization, options) + if err != nil { + return nil, err + } + + var names []string + for _, w := range ws { + if b.workspace != "" && w.Name == b.workspace { + names = append(names, backend.DefaultStateName) + continue + } + if b.prefix != "" && strings.HasPrefix(w.Name, b.prefix) { + names = append(names, strings.TrimPrefix(w.Name, b.prefix)) + } + } + + // Sort the result so we have consistent output. + sort.StringSlice(names).Sort() + + return names, nil +} + +// Operation implements backend.Enhanced +func (b *Remote) Operation(ctx context.Context, op *backend.Operation) (*backend.RunningOperation, error) { + // Configure the remote workspace name. + if op.Workspace == backend.DefaultStateName { + op.Workspace = b.workspace + } else if b.prefix != "" && !strings.HasPrefix(op.Workspace, b.prefix) { + op.Workspace = b.prefix + op.Workspace + } + + // Determine the function to call for our operation + var f func(context.Context, context.Context, *backend.Operation, *backend.RunningOperation) + switch op.Type { + case backend.OperationTypePlan: + f = b.opPlan + default: + return nil, fmt.Errorf( + "\n\nThe \"remote\" backend currently only supports the \"plan\" operation.\n"+ + "Please use the remote backend web UI for all other operations:\n"+ + "https://%s/app/%s/%s", b.hostname, b.organization, op.Workspace) + // return nil, backend.ErrOperationNotSupported + } + + // Lock + b.opLock.Lock() + + // Build our running operation + // the runninCtx is only used to block until the operation returns. + runningCtx, done := context.WithCancel(context.Background()) + runningOp := &backend.RunningOperation{ + Context: runningCtx, + } + + // stopCtx wraps the context passed in, and is used to signal a graceful Stop. + stopCtx, stop := context.WithCancel(ctx) + runningOp.Stop = stop + + // cancelCtx is used to cancel the operation immediately, usually + // indicating that the process is exiting. + cancelCtx, cancel := context.WithCancel(context.Background()) + runningOp.Cancel = cancel + + // Do it + go func() { + defer done() + defer stop() + defer cancel() + + defer b.opLock.Unlock() + f(stopCtx, cancelCtx, op, runningOp) + }() + + // Return + return runningOp, nil +} + +// Colorize returns the Colorize structure that can be used for colorizing +// output. This is gauranteed to always return a non-nil value and so is useful +// as a helper to wrap any potentially colored strings. +func (b *Remote) Colorize() *colorstring.Colorize { + if b.CLIColor != nil { + return b.CLIColor + } + + return &colorstring.Colorize{ + Colors: colorstring.DefaultColors, + Disable: true, + } +} + +const generalErr = ` +%s: %v + +The "remote" backend encountered an unexpected error while communicating +with remote backend. In some cases this could be caused by a network +connection problem, in which case you could retry the command. If the issue +persists please open a support ticket to get help resolving the problem. +` + +var schemaDescriptions = map[string]string{ + "hostname": "The remote backend hostname to connect to (defaults to app.terraform.io).", + "organization": "The name of the organization containing the targeted workspace(s).", + "token": "The token used to authenticate with the remote backend. If TFE_TOKEN is set\n" + + "or credentials for the host are configured in the CLI Config File, then this\n" + + "this will override any saved value for this.", + "workspaces": "Workspaces contains arguments used to filter down to a set of workspaces\n" + + "to work on.", + "name": "A workspace name used to map the default workspace to a named remote workspace.\n" + + "When configured only the default workspace can be used. This option conflicts\n" + + "with \"prefix\"", + "prefix": "A prefix used to filter workspaces using a single configuration. New workspaces\n" + + "will automatically be prefixed with this prefix. If omitted only the default\n" + + "workspace can be used. This option conflicts with \"name\"", +} diff --git a/backend/remote/backend_mock.go b/backend/remote/backend_mock.go new file mode 100644 index 000000000..9aea17168 --- /dev/null +++ b/backend/remote/backend_mock.go @@ -0,0 +1,384 @@ +package remote + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + + tfe "github.com/hashicorp/go-tfe" +) + +type mockConfigurationVersions struct { + configVersions map[string]*tfe.ConfigurationVersion + uploadURLs map[string]*tfe.ConfigurationVersion + workspaces map[string]*tfe.ConfigurationVersion +} + +func newMockConfigurationVersions() *mockConfigurationVersions { + return &mockConfigurationVersions{ + configVersions: make(map[string]*tfe.ConfigurationVersion), + uploadURLs: make(map[string]*tfe.ConfigurationVersion), + workspaces: make(map[string]*tfe.ConfigurationVersion), + } +} + +func (m *mockConfigurationVersions) List(ctx context.Context, workspaceID string, options tfe.ConfigurationVersionListOptions) ([]*tfe.ConfigurationVersion, error) { + var cvs []*tfe.ConfigurationVersion + for _, cv := range m.configVersions { + cvs = append(cvs, cv) + } + return cvs, nil +} + +func (m *mockConfigurationVersions) Create(ctx context.Context, workspaceID string, options tfe.ConfigurationVersionCreateOptions) (*tfe.ConfigurationVersion, error) { + id := generateID("cv-") + url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", id) + + cv := &tfe.ConfigurationVersion{ + ID: id, + Status: tfe.ConfigurationPending, + UploadURL: url, + } + + m.configVersions[cv.ID] = cv + m.uploadURLs[url] = cv + m.workspaces[workspaceID] = cv + + return cv, nil +} + +func (m *mockConfigurationVersions) Read(ctx context.Context, cvID string) (*tfe.ConfigurationVersion, error) { + cv, ok := m.configVersions[cvID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return cv, nil +} + +func (m *mockConfigurationVersions) Upload(ctx context.Context, url, path string) error { + cv, ok := m.uploadURLs[url] + if !ok { + return errors.New("404 not found") + } + cv.Status = tfe.ConfigurationUploaded + return nil +} + +type mockOrganizations struct { + organizations map[string]*tfe.Organization +} + +func newMockOrganizations() *mockOrganizations { + return &mockOrganizations{ + organizations: make(map[string]*tfe.Organization), + } +} + +func (m *mockOrganizations) List(ctx context.Context, options tfe.OrganizationListOptions) ([]*tfe.Organization, error) { + var orgs []*tfe.Organization + for _, org := range m.organizations { + orgs = append(orgs, org) + } + return orgs, nil +} + +func (m *mockOrganizations) Create(ctx context.Context, options tfe.OrganizationCreateOptions) (*tfe.Organization, error) { + org := &tfe.Organization{Name: *options.Name} + m.organizations[org.Name] = org + return org, nil +} + +func (m *mockOrganizations) Read(ctx context.Context, name string) (*tfe.Organization, error) { + org, ok := m.organizations[name] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return org, nil +} + +func (m *mockOrganizations) Update(ctx context.Context, name string, options tfe.OrganizationUpdateOptions) (*tfe.Organization, error) { + org, ok := m.organizations[name] + if !ok { + return nil, tfe.ErrResourceNotFound + } + org.Name = *options.Name + return org, nil + +} + +func (m *mockOrganizations) Delete(ctx context.Context, name string) error { + delete(m.organizations, name) + return nil +} + +type mockPlans struct { + logs map[string]string + plans map[string]*tfe.Plan +} + +func newMockPlans() *mockPlans { + return &mockPlans{ + logs: make(map[string]string), + plans: make(map[string]*tfe.Plan), + } +} + +func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) { + p, ok := m.plans[planID] + if !ok { + url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", planID) + + p = &tfe.Plan{ + ID: planID, + LogReadURL: url, + Status: tfe.PlanFinished, + } + + m.logs[url] = "plan/output.log" + m.plans[p.ID] = p + } + + return p, nil +} + +func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error) { + p, err := m.Read(ctx, planID) + if err != nil { + return nil, err + } + + logfile, ok := m.logs[p.LogReadURL] + if !ok { + return nil, tfe.ErrResourceNotFound + } + + logs, err := ioutil.ReadFile("./test-fixtures/" + logfile) + if err != nil { + return nil, err + } + + return bytes.NewBuffer(logs), nil +} + +type mockRuns struct { + runs map[string]*tfe.Run + workspaces map[string][]*tfe.Run +} + +func newMockRuns() *mockRuns { + return &mockRuns{ + runs: make(map[string]*tfe.Run), + workspaces: make(map[string][]*tfe.Run), + } +} + +func (m *mockRuns) List(ctx context.Context, workspaceID string, options tfe.RunListOptions) ([]*tfe.Run, error) { + var rs []*tfe.Run + for _, r := range m.workspaces[workspaceID] { + rs = append(rs, r) + } + return rs, nil +} + +func (m *mockRuns) Create(ctx context.Context, options tfe.RunCreateOptions) (*tfe.Run, error) { + id := generateID("run-") + p := &tfe.Plan{ + ID: generateID("plan-"), + Status: tfe.PlanPending, + } + + r := &tfe.Run{ + ID: id, + Plan: p, + Status: tfe.RunPending, + } + + m.runs[r.ID] = r + m.workspaces[options.Workspace.ID] = append(m.workspaces[options.Workspace.ID], r) + + return r, nil +} + +func (m *mockRuns) Read(ctx context.Context, runID string) (*tfe.Run, error) { + r, ok := m.runs[runID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return r, nil +} + +func (m *mockRuns) Apply(ctx context.Context, runID string, options tfe.RunApplyOptions) error { + panic("not implemented") +} + +func (m *mockRuns) Cancel(ctx context.Context, runID string, options tfe.RunCancelOptions) error { + panic("not implemented") +} + +func (m *mockRuns) Discard(ctx context.Context, runID string, options tfe.RunDiscardOptions) error { + panic("not implemented") +} + +type mockStateVersions struct { + states map[string][]byte + stateVersions map[string]*tfe.StateVersion + workspaces map[string][]string +} + +func newMockStateVersions() *mockStateVersions { + return &mockStateVersions{ + states: make(map[string][]byte), + stateVersions: make(map[string]*tfe.StateVersion), + workspaces: make(map[string][]string), + } +} + +func (m *mockStateVersions) List(ctx context.Context, options tfe.StateVersionListOptions) ([]*tfe.StateVersion, error) { + var svs []*tfe.StateVersion + for _, sv := range m.stateVersions { + svs = append(svs, sv) + } + return svs, nil +} + +func (m *mockStateVersions) Create(ctx context.Context, workspaceID string, options tfe.StateVersionCreateOptions) (*tfe.StateVersion, error) { + id := generateID("sv-") + url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", id) + + sv := &tfe.StateVersion{ + ID: id, + DownloadURL: url, + Serial: *options.Serial, + } + + state, err := base64.StdEncoding.DecodeString(*options.State) + if err != nil { + return nil, err + } + + m.states[sv.DownloadURL] = state + m.stateVersions[sv.ID] = sv + m.workspaces[workspaceID] = append(m.workspaces[workspaceID], sv.ID) + + return sv, nil +} + +func (m *mockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVersion, error) { + sv, ok := m.stateVersions[svID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return sv, nil +} + +func (m *mockStateVersions) Current(ctx context.Context, workspaceID string) (*tfe.StateVersion, error) { + svs, ok := m.workspaces[workspaceID] + if !ok || len(svs) == 0 { + return nil, tfe.ErrResourceNotFound + } + sv, ok := m.stateVersions[svs[len(svs)-1]] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return sv, nil +} + +func (m *mockStateVersions) Download(ctx context.Context, url string) ([]byte, error) { + state, ok := m.states[url] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return state, nil +} + +type mockWorkspaces struct { + workspaceIDs map[string]*tfe.Workspace + workspaceNames map[string]*tfe.Workspace +} + +func newMockWorkspaces() *mockWorkspaces { + return &mockWorkspaces{ + workspaceIDs: make(map[string]*tfe.Workspace), + workspaceNames: make(map[string]*tfe.Workspace), + } +} + +func (m *mockWorkspaces) List(ctx context.Context, organization string, options tfe.WorkspaceListOptions) ([]*tfe.Workspace, error) { + var ws []*tfe.Workspace + for _, w := range m.workspaceIDs { + ws = append(ws, w) + } + return ws, nil +} + +func (m *mockWorkspaces) Create(ctx context.Context, organization string, options tfe.WorkspaceCreateOptions) (*tfe.Workspace, error) { + id := generateID("ws-") + w := &tfe.Workspace{ + ID: id, + Name: *options.Name, + } + m.workspaceIDs[w.ID] = w + m.workspaceNames[w.Name] = w + return w, nil +} + +func (m *mockWorkspaces) Read(ctx context.Context, organization, workspace string) (*tfe.Workspace, error) { + w, ok := m.workspaceNames[workspace] + if !ok { + return nil, tfe.ErrResourceNotFound + } + return w, nil +} + +func (m *mockWorkspaces) Update(ctx context.Context, organization, workspace string, options tfe.WorkspaceUpdateOptions) (*tfe.Workspace, error) { + w, ok := m.workspaceNames[workspace] + if !ok { + return nil, tfe.ErrResourceNotFound + } + w.Name = *options.Name + w.TerraformVersion = *options.TerraformVersion + + delete(m.workspaceNames, workspace) + m.workspaceNames[w.Name] = w + + return w, nil +} + +func (m *mockWorkspaces) Delete(ctx context.Context, organization, workspace string) error { + if w, ok := m.workspaceNames[workspace]; ok { + delete(m.workspaceIDs, w.ID) + } + delete(m.workspaceNames, workspace) + return nil +} + +func (m *mockWorkspaces) Lock(ctx context.Context, workspaceID string, options tfe.WorkspaceLockOptions) (*tfe.Workspace, error) { + panic("not implemented") +} + +func (m *mockWorkspaces) Unlock(ctx context.Context, workspaceID string) (*tfe.Workspace, error) { + panic("not implemented") +} + +func (m *mockWorkspaces) AssignSSHKey(ctx context.Context, workspaceID string, options tfe.WorkspaceAssignSSHKeyOptions) (*tfe.Workspace, error) { + panic("not implemented") +} + +func (m *mockWorkspaces) UnassignSSHKey(ctx context.Context, workspaceID string) (*tfe.Workspace, error) { + panic("not implemented") +} + +const alphanumeric = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +func generateID(s string) string { + b := make([]byte, 16) + for i := range b { + b[i] = alphanumeric[rand.Intn(len(alphanumeric))] + } + return s + string(b) +} diff --git a/backend/remote/backend_plan.go b/backend/remote/backend_plan.go new file mode 100644 index 000000000..392830125 --- /dev/null +++ b/backend/remote/backend_plan.go @@ -0,0 +1,206 @@ +package remote + +import ( + "bufio" + "context" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strings" + "time" + + tfe "github.com/hashicorp/go-tfe" + "github.com/hashicorp/terraform/backend" +) + +func (b *Remote) opPlan(stopCtx, cancelCtx context.Context, op *backend.Operation, runningOp *backend.RunningOperation) { + log.Printf("[INFO] backend/remote: starting Plan operation") + + if op.Plan != nil { + runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrPlanNotSupported)) + return + } + + if op.PlanOutPath != "" { + runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrOutPathNotSupported)) + return + } + + if op.Targets != nil { + runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrTargetsNotSupported)) + return + } + + if (op.Module == nil || op.Module.Config().Dir == "") && !op.Destroy { + runningOp.Err = fmt.Errorf(strings.TrimSpace(planErrNoConfig)) + return + } + + // Retrieve the workspace used to run this operation in. + w, err := b.client.Workspaces.Read(stopCtx, b.organization, op.Workspace) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error retrieving workspace", err))) + } + return + } + + configOptions := tfe.ConfigurationVersionCreateOptions{ + AutoQueueRuns: tfe.Bool(false), + Speculative: tfe.Bool(true), + } + + cv, err := b.client.ConfigurationVersions.Create(stopCtx, w.ID, configOptions) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error creating configuration version", err))) + } + return + } + + var configDir string + if op.Module != nil && op.Module.Config().Dir != "" { + configDir = op.Module.Config().Dir + } else { + configDir, err = ioutil.TempDir("", "tf") + if err != nil { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error creating temp directory", err))) + return + } + defer os.RemoveAll(configDir) + } + + err = b.client.ConfigurationVersions.Upload(stopCtx, cv.UploadURL, configDir) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error uploading configuration files", err))) + } + return + } + + uploaded := false + for i := 0; i < 60 && !uploaded; i++ { + select { + case <-stopCtx.Done(): + return + case <-cancelCtx.Done(): + return + case <-time.After(500 * time.Millisecond): + cv, err = b.client.ConfigurationVersions.Read(stopCtx, cv.ID) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error retrieving configuration version", err))) + } + return + } + + if cv.Status == tfe.ConfigurationUploaded { + uploaded = true + } + } + } + + if !uploaded { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error uploading configuration files", "operation timed out"))) + return + } + + runOptions := tfe.RunCreateOptions{ + IsDestroy: tfe.Bool(op.Destroy), + Message: tfe.String("Queued manually using Terraform"), + ConfigurationVersion: cv, + Workspace: w, + } + + r, err := b.client.Runs.Create(stopCtx, runOptions) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error creating run", err))) + } + return + } + + r, err = b.client.Runs.Read(stopCtx, r.ID) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error retrieving run", err))) + } + return + } + + if b.CLI != nil { + b.CLI.Output(b.Colorize().Color(strings.TrimSpace(fmt.Sprintf( + planDefaultHeader, b.hostname, b.organization, op.Workspace, r.ID)) + "\n")) + } + + logs, err := b.client.Plans.Logs(stopCtx, r.Plan.ID) + if err != nil { + if err != context.Canceled { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error retrieving logs", err))) + } + return + } + scanner := bufio.NewScanner(logs) + + for scanner.Scan() { + if b.CLI != nil { + b.CLI.Output(b.Colorize().Color(scanner.Text())) + } + } + if err := scanner.Err(); err != nil { + if err != context.Canceled && err != io.EOF { + runningOp.Err = fmt.Errorf("Error reading logs: %v", err) + } + return + } +} + +const planErrPlanNotSupported = ` +Displaying a saved plan is currently not supported! + +The "remote" backend currently requires configuration to be present +and does not accept an existing saved plan as an argument at this time. +` + +const planErrOutPathNotSupported = ` +Saving a generated plan is currently not supported! + +The "remote" backend does not support saving the generated execution +plan locally at this time. +` + +const planErrTargetsNotSupported = ` +Resource targeting is currently not supported! + +The "remote" backend does not support resource targeting at this time. +` + +const planErrNoConfig = ` +No configuration files found! + +Plan requires configuration to be present. Planning without a configuration +would mark everything for destruction, which is normally not what is desired. +If you would like to destroy everything, please run plan with the "-destroy" +flag or create a single empty configuration file. Otherwise, please create +a Terraform configuration file in the path being executed and try again. +` + +const planDefaultHeader = ` +[reset][yellow]Running plan in the remote backend. Output will stream here. Pressing Ctrl-C +will stop streaming the logs, but will not stop the plan running remotely. +To view this plan in a browser, visit: +https://%s/app/%s/%s/runs/%s[reset] + +Waiting for the plan to start... +` diff --git a/backend/remote/backend_plan_test.go b/backend/remote/backend_plan_test.go new file mode 100644 index 000000000..cf3405729 --- /dev/null +++ b/backend/remote/backend_plan_test.go @@ -0,0 +1,181 @@ +package remote + +import ( + "context" + "strings" + "testing" + + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/config/module" + "github.com/hashicorp/terraform/terraform" + "github.com/mitchellh/cli" +) + +func testOperationPlan() *backend.Operation { + return &backend.Operation{ + Type: backend.OperationTypePlan, + } +} + +func TestRemote_planBasic(t *testing.T) { + b := testBackendDefault(t) + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + + <-run.Done() + if run.Err != nil { + t.Fatalf("error running operation: %v", run.Err) + } + + output := b.CLI.(*cli.MockUi).OutputWriter.String() + if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") { + t.Fatalf("missing plan summery in output: %s", output) + } +} + +func TestRemote_planWithPlan(t *testing.T) { + b := testBackendDefault(t) + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.Plan = &terraform.Plan{} + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + <-run.Done() + + if run.Err == nil { + t.Fatalf("expected a plan error, got: %v", run.Err) + } + if !strings.Contains(run.Err.Error(), "saved plan is currently not supported") { + t.Fatalf("expected a saved plan error, got: %v", run.Err) + } +} + +func TestRemote_planWithPath(t *testing.T) { + b := testBackendDefault(t) + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.PlanOutPath = "./test-fixtures/plan" + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + <-run.Done() + + if run.Err == nil { + t.Fatalf("expected a plan error, got: %v", run.Err) + } + if !strings.Contains(run.Err.Error(), "generated plan is currently not supported") { + t.Fatalf("expected a generated plan error, got: %v", run.Err) + } +} + +func TestRemote_planWithTarget(t *testing.T) { + b := testBackendDefault(t) + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.Targets = []string{"null_resource.foo"} + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + <-run.Done() + + if run.Err == nil { + t.Fatalf("expected a plan error, got: %v", run.Err) + } + if !strings.Contains(run.Err.Error(), "targeting is currently not supported") { + t.Fatalf("expected a targeting error, got: %v", run.Err) + } +} + +func TestRemote_planNoConfig(t *testing.T) { + b := testBackendDefault(t) + + op := testOperationPlan() + op.Module = nil + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + <-run.Done() + + if run.Err == nil { + t.Fatalf("expected a plan error, got: %v", run.Err) + } + if !strings.Contains(run.Err.Error(), "configuration files found") { + t.Fatalf("expected configuration files error, got: %v", run.Err) + } +} + +func TestRemote_planDestroy(t *testing.T) { + b := testBackendDefault(t) + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan") + defer modCleanup() + + op := testOperationPlan() + op.Destroy = true + op.Module = mod + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + + <-run.Done() + if run.Err != nil { + t.Fatalf("unexpected plan error: %v", run.Err) + } +} + +func TestRemote_planDestroyNoConfig(t *testing.T) { + b := testBackendDefault(t) + + op := testOperationPlan() + op.Destroy = true + op.Module = nil + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + + <-run.Done() + if run.Err != nil { + t.Fatalf("unexpected plan error: %v", run.Err) + } +} diff --git a/backend/remote/backend_state.go b/backend/remote/backend_state.go new file mode 100644 index 000000000..135d48a6d --- /dev/null +++ b/backend/remote/backend_state.go @@ -0,0 +1,103 @@ +package remote + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "fmt" + + tfe "github.com/hashicorp/go-tfe" + "github.com/hashicorp/terraform/state/remote" + "github.com/hashicorp/terraform/terraform" +) + +type remoteClient struct { + client *tfe.Client + organization string + workspace string +} + +// Get the remote state. +func (r *remoteClient) Get() (*remote.Payload, error) { + ctx := context.Background() + + // Retrieve the workspace for which to create a new state. + w, err := r.client.Workspaces.Read(ctx, r.organization, r.workspace) + if err != nil { + if err == tfe.ErrResourceNotFound { + // If no state exists, then return nil. + return nil, nil + } + return nil, fmt.Errorf("Error retrieving workspace: %v", err) + } + + sv, err := r.client.StateVersions.Current(ctx, w.ID) + if err != nil { + if err == tfe.ErrResourceNotFound { + // If no state exists, then return nil. + return nil, nil + } + return nil, fmt.Errorf("Error retrieving remote state: %v", err) + } + + state, err := r.client.StateVersions.Download(ctx, sv.DownloadURL) + if err != nil { + return nil, fmt.Errorf("Error downloading remote state: %v", err) + } + + // If the state is empty, then return nil. + if len(state) == 0 { + return nil, nil + } + + // Get the MD5 checksum of the state. + sum := md5.Sum(state) + + return &remote.Payload{ + Data: state, + MD5: sum[:], + }, nil +} + +// Put the remote state. +func (r *remoteClient) Put(state []byte) error { + ctx := context.Background() + + // Retrieve the workspace for which to create a new state. + w, err := r.client.Workspaces.Read(ctx, r.organization, r.workspace) + if err != nil { + return fmt.Errorf("Error retrieving workspace: %v", err) + } + + // the state into a buffer. + tfState, err := terraform.ReadState(bytes.NewReader(state)) + if err != nil { + return fmt.Errorf("Error reading state: %s", err) + } + + options := tfe.StateVersionCreateOptions{ + Lineage: tfe.String(tfState.Lineage), + Serial: tfe.Int64(tfState.Serial), + MD5: tfe.String(fmt.Sprintf("%x", md5.Sum(state))), + State: tfe.String(base64.StdEncoding.EncodeToString(state)), + } + + // Create the new state. + _, err = r.client.StateVersions.Create(ctx, w.ID, options) + if err != nil { + return fmt.Errorf("Error creating remote state: %v", err) + } + + return nil +} + +// Delete the remote state. +func (r *remoteClient) Delete() error { + err := r.client.Workspaces.Delete(context.Background(), r.organization, r.workspace) + if err != nil && err != tfe.ErrResourceNotFound { + return fmt.Errorf("Error deleting workspace %s: %v", r.workspace, err) + } + + return nil +} diff --git a/backend/remote/backend_state_test.go b/backend/remote/backend_state_test.go new file mode 100644 index 000000000..d3c4478a0 --- /dev/null +++ b/backend/remote/backend_state_test.go @@ -0,0 +1,16 @@ +package remote + +import ( + "testing" + + "github.com/hashicorp/terraform/state/remote" +) + +func TestRemoteClient_impl(t *testing.T) { + var _ remote.Client = new(remoteClient) +} + +func TestRemoteClient(t *testing.T) { + client := testRemoteClient(t) + remote.TestClient(t, client) +} diff --git a/backend/remote/backend_test.go b/backend/remote/backend_test.go new file mode 100644 index 000000000..895a7fd55 --- /dev/null +++ b/backend/remote/backend_test.go @@ -0,0 +1,254 @@ +package remote + +import ( + "errors" + "reflect" + "strings" + "testing" + + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/config" + "github.com/hashicorp/terraform/terraform" +) + +func TestRemote(t *testing.T) { + var _ backend.Enhanced = New(nil) + var _ backend.CLI = New(nil) +} + +func TestRemote_config(t *testing.T) { + cases := map[string]struct { + config map[string]interface{} + err error + }{ + "with_a_name": { + config: map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "name": "prod", + }, + }, + }, + err: nil, + }, + "with_a_prefix": { + config: map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "prefix": "my-app-", + }, + }, + }, + err: nil, + }, + "with_two_workspace_entries": { + config: map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "name": "prod", + }, + map[string]interface{}{ + "prefix": "my-app-", + }, + }, + }, + err: errors.New("only one 'workspaces' block allowed"), + }, + "without_either_a_name_and_a_prefix": { + config: map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{}, + }, + }, + err: errors.New("either workspace 'name' or 'prefix' is required"), + }, + "with_both_a_name_and_a_prefix": { + config: map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "name": "prod", + "prefix": "my-app-", + }, + }, + }, + err: errors.New("only one of workspace 'name' or 'prefix' is allowed"), + }, + "with_an_unknown_host": { + config: map[string]interface{}{ + "hostname": "nonexisting.local", + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "name": "prod", + }, + }, + }, + err: errors.New("host nonexisting.local does not provide a remote backend API"), + }, + } + + for name, tc := range cases { + s := testServer(t) + b := New(testDisco(s)) + + // Get the proper config structure + rc, err := config.NewRawConfig(tc.config) + if err != nil { + t.Fatalf("%s: error creating raw config: %v", name, err) + } + conf := terraform.NewResourceConfig(rc) + + // Validate + warns, errs := b.Validate(conf) + if len(warns) > 0 { + t.Fatalf("%s: validation warnings: %v", name, warns) + } + if len(errs) > 0 { + t.Fatalf("%s: validation errors: %v", name, errs) + } + + // Configure + err = b.Configure(conf) + if err != tc.err && err != nil && tc.err != nil && err.Error() != tc.err.Error() { + t.Fatalf("%s: expected error %q, got: %q", name, tc.err, err) + } + } +} + +func TestRemote_nonexistingOrganization(t *testing.T) { + msg := "does not exist" + + b := testBackendNoDefault(t) + b.organization = "nonexisting" + + if _, err := b.State("prod"); err == nil || !strings.Contains(err.Error(), msg) { + t.Fatalf("expected %q error, got: %v", msg, err) + } + + if err := b.DeleteState("prod"); err == nil || !strings.Contains(err.Error(), msg) { + t.Fatalf("expected %q error, got: %v", msg, err) + } + + if _, err := b.States(); err == nil || !strings.Contains(err.Error(), msg) { + t.Fatalf("expected %q error, got: %v", msg, err) + } +} + +func TestRemote_backendDefault(t *testing.T) { + b := testBackendDefault(t) + backend.TestBackendStates(t, b) + backend.TestBackendStateLocks(t, b, b) + backend.TestBackendStateForceUnlock(t, b, b) +} + +func TestRemote_backendNoDefault(t *testing.T) { + b := testBackendNoDefault(t) + backend.TestBackendStates(t, b) +} + +func TestRemote_addAndRemoveStatesDefault(t *testing.T) { + b := testBackendDefault(t) + if _, err := b.States(); err != backend.ErrNamedStatesNotSupported { + t.Fatalf("expected error %v, got %v", backend.ErrNamedStatesNotSupported, err) + } + + if _, err := b.State(backend.DefaultStateName); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if _, err := b.State("prod"); err != backend.ErrNamedStatesNotSupported { + t.Fatalf("expected error %v, got %v", backend.ErrNamedStatesNotSupported, err) + } + + if err := b.DeleteState(backend.DefaultStateName); err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if err := b.DeleteState("prod"); err != backend.ErrNamedStatesNotSupported { + t.Fatalf("expected error %v, got %v", backend.ErrNamedStatesNotSupported, err) + } +} + +func TestRemote_addAndRemoveStatesNoDefault(t *testing.T) { + b := testBackendNoDefault(t) + states, err := b.States() + if err != nil { + t.Fatal(err) + } + + expectedStates := []string(nil) + if !reflect.DeepEqual(states, expectedStates) { + t.Fatalf("expected states %#+v, got %#+v", expectedStates, states) + } + + if _, err := b.State(backend.DefaultStateName); err != backend.ErrDefaultStateNotSupported { + t.Fatalf("expected error %v, got %v", backend.ErrDefaultStateNotSupported, err) + } + + expectedA := "test_A" + if _, err := b.State(expectedA); err != nil { + t.Fatal(err) + } + + states, err = b.States() + if err != nil { + t.Fatal(err) + } + + expectedStates = append(expectedStates, expectedA) + if !reflect.DeepEqual(states, expectedStates) { + t.Fatalf("expected %#+v, got %#+v", expectedStates, states) + } + + expectedB := "test_B" + if _, err := b.State(expectedB); err != nil { + t.Fatal(err) + } + + states, err = b.States() + if err != nil { + t.Fatal(err) + } + + expectedStates = append(expectedStates, expectedB) + if !reflect.DeepEqual(states, expectedStates) { + t.Fatalf("expected %#+v, got %#+v", expectedStates, states) + } + + if err := b.DeleteState(backend.DefaultStateName); err != backend.ErrDefaultStateNotSupported { + t.Fatalf("expected error %v, got %v", backend.ErrDefaultStateNotSupported, err) + } + + if err := b.DeleteState(expectedA); err != nil { + t.Fatal(err) + } + + states, err = b.States() + if err != nil { + t.Fatal(err) + } + + expectedStates = []string{expectedB} + if !reflect.DeepEqual(states, expectedStates) { + t.Fatalf("expected %#+v got %#+v", expectedStates, states) + } + + if err := b.DeleteState(expectedB); err != nil { + t.Fatal(err) + } + + states, err = b.States() + if err != nil { + t.Fatal(err) + } + + expectedStates = []string(nil) + if !reflect.DeepEqual(states, expectedStates) { + t.Fatalf("expected %#+v, got %#+v", expectedStates, states) + } +} diff --git a/backend/remote/cli.go b/backend/remote/cli.go new file mode 100644 index 000000000..9339c1091 --- /dev/null +++ b/backend/remote/cli.go @@ -0,0 +1,13 @@ +package remote + +import ( + "github.com/hashicorp/terraform/backend" +) + +// CLIInit implements backend.CLI +func (b *Remote) CLIInit(opts *backend.CLIOpts) error { + b.CLI = opts.CLI + b.CLIColor = opts.CLIColor + b.ContextOpts = opts.ContextOpts + return nil +} diff --git a/backend/remote/test-fixtures/plan-scaleout/main.tf b/backend/remote/test-fixtures/plan-scaleout/main.tf new file mode 100644 index 000000000..4067af592 --- /dev/null +++ b/backend/remote/test-fixtures/plan-scaleout/main.tf @@ -0,0 +1,10 @@ +resource "test_instance" "foo" { + count = 3 + ami = "bar" + + # This is here because at some point it caused a test failure + network_interface { + device_index = 0 + description = "Main network interface" + } +} diff --git a/backend/remote/test-fixtures/plan/main.tf b/backend/remote/test-fixtures/plan/main.tf new file mode 100644 index 000000000..3911a2a9b --- /dev/null +++ b/backend/remote/test-fixtures/plan/main.tf @@ -0,0 +1 @@ +resource "null_resource" "foo" {} diff --git a/backend/remote/test-fixtures/plan/output.log b/backend/remote/test-fixtures/plan/output.log new file mode 100644 index 000000000..d9fe98082 --- /dev/null +++ b/backend/remote/test-fixtures/plan/output.log @@ -0,0 +1,29 @@ +Running plan in the remote backend. Output will stream here. Pressing Ctrl-C +will stop streaming the logs, but will not stop the plan running remotely. +To view this plan in a browser, visit: +https://atlas.local/app/demo1/my-app-web/runs/run-cPK6EnfTpqwy6ucU + +Waiting for the plan to start... + +Terraform v0.11.7 + +Configuring remote state backend... +Initializing Terraform configuration... +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + + null_resource.foo + id: + + +Plan: 1 to add, 0 to change, 0 to destroy. diff --git a/backend/remote/testing.go b/backend/remote/testing.go new file mode 100644 index 000000000..253272f76 --- /dev/null +++ b/backend/remote/testing.go @@ -0,0 +1,128 @@ +package remote + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + + tfe "github.com/hashicorp/go-tfe" + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/state/remote" + "github.com/hashicorp/terraform/svchost" + "github.com/hashicorp/terraform/svchost/auth" + "github.com/hashicorp/terraform/svchost/disco" + "github.com/mitchellh/cli" +) + +const ( + testCred = "test-auth-token" +) + +var ( + tfeHost = svchost.Hostname(defaultHostname) + credsSrc = auth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{ + tfeHost: {"token": testCred}, + }) +) + +func testBackendDefault(t *testing.T) *Remote { + c := map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "name": "prod", + }, + }, + } + return testBackend(t, c) +} + +func testBackendNoDefault(t *testing.T) *Remote { + c := map[string]interface{}{ + "organization": "hashicorp", + "workspaces": []interface{}{ + map[string]interface{}{ + "prefix": "my-app-", + }, + }, + } + return testBackend(t, c) +} + +func testRemoteClient(t *testing.T) remote.Client { + b := testBackendDefault(t) + raw, err := b.State(backend.DefaultStateName) + if err != nil { + t.Fatalf("error: %v", err) + } + s := raw.(*remote.State) + return s.Client +} + +func testBackend(t *testing.T, c map[string]interface{}) *Remote { + s := testServer(t) + b := New(testDisco(s)) + + // Configure the backend so the client is created. + backend.TestBackendConfig(t, b, c) + + // Once the client exists, mock the services we use.. + b.CLI = cli.NewMockUi() + b.client.ConfigurationVersions = newMockConfigurationVersions() + b.client.Organizations = newMockOrganizations() + b.client.Plans = newMockPlans() + b.client.Runs = newMockRuns() + b.client.StateVersions = newMockStateVersions() + b.client.Workspaces = newMockWorkspaces() + + ctx := context.Background() + + // Create the organization. + _, err := b.client.Organizations.Create(ctx, tfe.OrganizationCreateOptions{ + Name: tfe.String(b.organization), + }) + if err != nil { + t.Fatalf("error: %v", err) + } + + // Create the default workspace if required. + if b.workspace != "" { + _, err = b.client.Workspaces.Create(ctx, b.organization, tfe.WorkspaceCreateOptions{ + Name: tfe.String(b.workspace), + }) + if err != nil { + t.Fatalf("error: %v", err) + } + } + + return b +} + +// testServer returns a *httptest.Server used for local testing. +func testServer(t *testing.T) *httptest.Server { + mux := http.NewServeMux() + + // Respond to service discovery calls. + mux.HandleFunc("/well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"tfe.v2":"/api/v2/"}`) + }) + + return httptest.NewServer(mux) +} + +// testDisco returns a *disco.Disco mapping app.terraform.io and +// localhost to a local test server. +func testDisco(s *httptest.Server) *disco.Disco { + services := map[string]interface{}{ + "tfe.v2": fmt.Sprintf("%s/api/v2/", s.URL), + } + d := disco.NewWithCredentialsSource(credsSrc) + + d.ForceHostServices(svchost.Hostname(defaultHostname), services) + d.ForceHostServices(svchost.Hostname("localhost"), services) + return d +} diff --git a/backend/testing.go b/backend/testing.go index e509f82c2..22dc99791 100644 --- a/backend/testing.go +++ b/backend/testing.go @@ -47,15 +47,27 @@ func TestBackendConfig(t *testing.T, b Backend, c map[string]interface{}) Backen func TestBackendStates(t *testing.T, b Backend) { t.Helper() + noDefault := false + if _, err := b.State(DefaultStateName); err != nil { + if err == ErrDefaultStateNotSupported { + noDefault = true + } else { + t.Fatalf("error: %v", err) + } + } + states, err := b.States() - if err == ErrNamedStatesNotSupported { - t.Logf("TestBackend: named states not supported in %T, skipping", b) - return + if err != nil { + if err == ErrNamedStatesNotSupported { + t.Logf("TestBackend: named states not supported in %T, skipping", b) + return + } + t.Fatalf("error: %v", err) } // Test it starts with only the default - if len(states) != 1 || states[0] != DefaultStateName { - t.Fatalf("should only have default to start: %#v", states) + if !noDefault && (len(states) != 1 || states[0] != DefaultStateName) { + t.Fatalf("should have default to start: %#v", states) } // Create a couple states @@ -175,6 +187,9 @@ func TestBackendStates(t *testing.T, b Backend) { sort.Strings(states) expected := []string{"bar", "default", "foo"} + if noDefault { + expected = []string{"bar", "foo"} + } if !reflect.DeepEqual(states, expected) { t.Fatalf("bad: %#v", states) } @@ -218,6 +233,9 @@ func TestBackendStates(t *testing.T, b Backend) { sort.Strings(states) expected := []string{"bar", "default"} + if noDefault { + expected = []string{"bar"} + } if !reflect.DeepEqual(states, expected) { t.Fatalf("bad: %#v", states) } diff --git a/builtin/providers/terraform/provider_test.go b/builtin/providers/terraform/provider_test.go index 65f3ce4ad..0ba389b5c 100644 --- a/builtin/providers/terraform/provider_test.go +++ b/builtin/providers/terraform/provider_test.go @@ -3,6 +3,7 @@ package terraform import ( "testing" + backendInit "github.com/hashicorp/terraform/backend/init" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/terraform" ) @@ -11,6 +12,9 @@ var testAccProviders map[string]terraform.ResourceProvider var testAccProvider *schema.Provider func init() { + // Initialize the backends + backendInit.Init(nil) + testAccProvider = Provider().(*schema.Provider) testAccProviders = map[string]terraform.ResourceProvider{ "terraform": testAccProvider, diff --git a/command/command.go b/command/command.go index 0cd11da08..815a6fa6d 100644 --- a/command/command.go +++ b/command/command.go @@ -48,10 +48,6 @@ The "backend" in Terraform defines how Terraform operates. The default backend performs all operations locally on your machine. Your configuration is configured to use a non-local backend. This backend doesn't support this operation. - -If you want to use the state from the backend but force all other data -(configuration, variables, etc.) to come locally, you can force local -behavior with the "-local" flag. ` // ModulePath returns the path to the root module from the CLI args. diff --git a/command/command_test.go b/command/command_test.go index c0a8529c6..79330f42b 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -19,6 +19,7 @@ import ( "syscall" "testing" + backendInit "github.com/hashicorp/terraform/backend/init" "github.com/hashicorp/terraform/config/module" "github.com/hashicorp/terraform/helper/logging" "github.com/hashicorp/terraform/terraform" @@ -33,6 +34,9 @@ var testingDir string func init() { test = true + // Initialize the backends + backendInit.Init(nil) + // Expand the fixture dir on init because we change the working // directory in some tests. var err error diff --git a/command/init.go b/command/init.go index b96cdc6ec..c831566d9 100644 --- a/command/init.go +++ b/command/init.go @@ -140,8 +140,7 @@ func (c *InitCommand) Run(args []string) int { // the backend with an empty directory. empty, err := config.IsEmptyDir(path) if err != nil { - c.Ui.Error(fmt.Sprintf( - "Error checking configuration: %s", err)) + c.Ui.Error(fmt.Sprintf("Error checking configuration: %s", err)) return 1 } if empty { @@ -229,14 +228,12 @@ func (c *InitCommand) Run(args []string) int { if back != nil { sMgr, err := back.State(c.Workspace()) if err != nil { - c.Ui.Error(fmt.Sprintf( - "Error loading state: %s", err)) + c.Ui.Error(fmt.Sprintf("Error loading state: %s", err)) return 1 } if err := sMgr.RefreshState(); err != nil { - c.Ui.Error(fmt.Sprintf( - "Error refreshing state: %s", err)) + c.Ui.Error(fmt.Sprintf("Error refreshing state: %s", err)) return 1 } diff --git a/command/meta_backend_migrate.go b/command/meta_backend_migrate.go index 0c3610a8a..dc8d503b2 100644 --- a/command/meta_backend_migrate.go +++ b/command/meta_backend_migrate.go @@ -211,6 +211,13 @@ func (m *Meta) backendMigrateState_s_s(opts *backendMigrateOpts) error { stateTwo, err := opts.Two.State(opts.twoEnv) if err != nil { + if err == backend.ErrDefaultStateNotSupported && stateOne.State() == nil { + // When using named workspaces it is common that the default + // workspace is not actually used. So we first check if there + // actually is a state to be migrated, if not we just return + // and silently ignore the unused default worksopace. + return nil + } return fmt.Errorf(strings.TrimSpace( errMigrateSingleLoadDefault), opts.TwoType, err) } @@ -418,8 +425,8 @@ above error and try again. ` const errMigrateMulti = ` -Error migrating the workspace %q from the previous %q backend to the newly -configured %q backend: +Error migrating the workspace %q from the previous %q backend +to the newly configured %q backend: %s Terraform copies workspaces in alphabetical order. Any workspaces @@ -432,7 +439,8 @@ This will attempt to copy (with permission) all workspaces again. ` const errBackendStateCopy = ` -Error copying state from the previous %q backend to the newly configured %q backend: +Error copying state from the previous %q backend to the newly configured +%q backend: %s The state in the previous backend remains intact and unmodified. Please resolve diff --git a/command/meta_backend_test.go b/command/meta_backend_test.go index e3e735fdd..8e3059379 100644 --- a/command/meta_backend_test.go +++ b/command/meta_backend_test.go @@ -1422,6 +1422,112 @@ func TestMetaBackend_configuredChangeCopy_multiToMulti(t *testing.T) { } } +// Changing a configured backend that supports multi-state to a +// backend that also supports multi-state, but doesn't allow a +// default state while the default state is non-empty. +func TestMetaBackend_configuredChangeCopy_multiToNoDefaultWithDefault(t *testing.T) { + // Create a temporary working directory that is empty + td := tempDir(t) + copy.CopyDir(testFixturePath("backend-change-multi-to-no-default-with-default"), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + // Register the single-state backend + backendInit.Set("local-no-default", backendLocal.TestNewLocalNoDefault) + defer backendInit.Set("local-no-default", nil) + + // Ask input + defer testInputMap(t, map[string]string{ + "backend-migrate-to-new": "yes", + "backend-migrate-multistate-to-multistate": "yes", + })() + + // Setup the meta + m := testMetaBackend(t, nil) + + // Get the backend + _, err := m.Backend(&BackendOpts{Init: true}) + if err == nil || !strings.Contains(err.Error(), "default state not supported") { + t.Fatalf("expected error to contain %q\ngot: %s", "default state not supported", err) + } +} + +// Changing a configured backend that supports multi-state to a +// backend that also supports multi-state, but doesn't allow a +// default state while the default state is empty. +func TestMetaBackend_configuredChangeCopy_multiToNoDefaultWithoutDefault(t *testing.T) { + // Create a temporary working directory that is empty + td := tempDir(t) + copy.CopyDir(testFixturePath("backend-change-multi-to-no-default-without-default"), td) + defer os.RemoveAll(td) + defer testChdir(t, td)() + + // Register the single-state backend + backendInit.Set("local-no-default", backendLocal.TestNewLocalNoDefault) + defer backendInit.Set("local-no-default", nil) + + // Ask input + defer testInputMap(t, map[string]string{ + "backend-migrate-to-new": "yes", + "backend-migrate-multistate-to-multistate": "yes", + })() + + // Setup the meta + m := testMetaBackend(t, nil) + + // Get the backend + b, err := m.Backend(&BackendOpts{Init: true}) + if err != nil { + t.Fatalf("bad: %s", err) + } + + // Check resulting states + states, err := b.States() + if err != nil { + t.Fatalf("bad: %s", err) + } + + sort.Strings(states) + expected := []string{"env2"} + if !reflect.DeepEqual(states, expected) { + t.Fatalf("bad: %#v", states) + } + + { + // Check the named state + s, err := b.State("env2") + if err != nil { + t.Fatalf("bad: %s", err) + } + if err := s.RefreshState(); err != nil { + t.Fatalf("bad: %s", err) + } + state := s.State() + if state == nil { + t.Fatal("state should not be nil") + } + if state.Lineage != "backend-change-env2" { + t.Fatalf("bad: %#v", state) + } + } + + { + // Verify existing workspaces exist + envPath := filepath.Join(backendLocal.DefaultWorkspaceDir, "env2", backendLocal.DefaultStateFilename) + if _, err := os.Stat(envPath); err != nil { + t.Fatal("env should exist") + } + } + + { + // Verify new workspaces exist + envPath := filepath.Join("envdir-new", "env2", backendLocal.DefaultStateFilename) + if _, err := os.Stat(envPath); err != nil { + t.Fatal("env should exist") + } + } +} + // Unsetting a saved backend func TestMetaBackend_configuredUnset(t *testing.T) { // Create a temporary working directory that is empty diff --git a/command/test-fixtures/backend-change-multi-to-no-default-with-default/.terraform/terraform.tfstate b/command/test-fixtures/backend-change-multi-to-no-default-with-default/.terraform/terraform.tfstate new file mode 100644 index 000000000..073bd7a82 --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-with-default/.terraform/terraform.tfstate @@ -0,0 +1,22 @@ +{ + "version": 3, + "serial": 0, + "lineage": "666f9301-7e65-4b19-ae23-71184bb19b03", + "backend": { + "type": "local", + "config": { + "path": "local-state.tfstate" + }, + "hash": 9073424445967744180 + }, + "modules": [ + { + "path": [ + "root" + ], + "outputs": {}, + "resources": {}, + "depends_on": [] + } + ] +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-with-default/local-state.tfstate b/command/test-fixtures/backend-change-multi-to-no-default-with-default/local-state.tfstate new file mode 100644 index 000000000..88c1d86ec --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-with-default/local-state.tfstate @@ -0,0 +1,6 @@ +{ + "version": 3, + "terraform_version": "0.8.2", + "serial": 7, + "lineage": "backend-change" +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-with-default/main.tf b/command/test-fixtures/backend-change-multi-to-no-default-with-default/main.tf new file mode 100644 index 000000000..93c5bced0 --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-with-default/main.tf @@ -0,0 +1,5 @@ +terraform { + backend "local-no-default" { + environment_dir = "envdir-new" + } +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-with-default/terraform.tfstate.d/env2/terraform.tfstate b/command/test-fixtures/backend-change-multi-to-no-default-with-default/terraform.tfstate.d/env2/terraform.tfstate new file mode 100644 index 000000000..855a27f4c --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-with-default/terraform.tfstate.d/env2/terraform.tfstate @@ -0,0 +1,6 @@ +{ + "version": 3, + "terraform_version": "0.8.2", + "serial": 7, + "lineage": "backend-change-env2" +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-without-default/.terraform/terraform.tfstate b/command/test-fixtures/backend-change-multi-to-no-default-without-default/.terraform/terraform.tfstate new file mode 100644 index 000000000..073bd7a82 --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-without-default/.terraform/terraform.tfstate @@ -0,0 +1,22 @@ +{ + "version": 3, + "serial": 0, + "lineage": "666f9301-7e65-4b19-ae23-71184bb19b03", + "backend": { + "type": "local", + "config": { + "path": "local-state.tfstate" + }, + "hash": 9073424445967744180 + }, + "modules": [ + { + "path": [ + "root" + ], + "outputs": {}, + "resources": {}, + "depends_on": [] + } + ] +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-without-default/main.tf b/command/test-fixtures/backend-change-multi-to-no-default-without-default/main.tf new file mode 100644 index 000000000..93c5bced0 --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-without-default/main.tf @@ -0,0 +1,5 @@ +terraform { + backend "local-no-default" { + environment_dir = "envdir-new" + } +} diff --git a/command/test-fixtures/backend-change-multi-to-no-default-without-default/terraform.tfstate.d/env2/terraform.tfstate b/command/test-fixtures/backend-change-multi-to-no-default-without-default/terraform.tfstate.d/env2/terraform.tfstate new file mode 100644 index 000000000..855a27f4c --- /dev/null +++ b/command/test-fixtures/backend-change-multi-to-no-default-without-default/terraform.tfstate.d/env2/terraform.tfstate @@ -0,0 +1,6 @@ +{ + "version": 3, + "terraform_version": "0.8.2", + "serial": 7, + "lineage": "backend-change-env2" +} diff --git a/main.go b/main.go index 523863e7b..108f49035 100644 --- a/main.go +++ b/main.go @@ -11,9 +11,8 @@ import ( "strings" "sync" - "github.com/mitchellh/colorstring" - "github.com/hashicorp/go-plugin" + backendInit "github.com/hashicorp/terraform/backend/init" "github.com/hashicorp/terraform/command/format" "github.com/hashicorp/terraform/helper/logging" "github.com/hashicorp/terraform/svchost/disco" @@ -21,6 +20,7 @@ import ( "github.com/mattn/go-colorable" "github.com/mattn/go-shellwords" "github.com/mitchellh/cli" + "github.com/mitchellh/colorstring" "github.com/mitchellh/panicwrap" "github.com/mitchellh/prefixedio" ) @@ -143,10 +143,16 @@ func wrappedMain() int { } } + // Get any configured credentials from the config and initialize + // a service discovery object. + credsSrc := credentialsSource(config) + services := disco.NewWithCredentialsSource(credsSrc) + + // Initialize the backends. + backendInit.Init(services) + // In tests, Commands may already be set to provide mock commands if Commands == nil { - credsSrc := credentialsSource(config) - services := disco.NewWithCredentialsSource(credsSrc) initCommands(config, services) } diff --git a/state/remote/testing.go b/state/remote/testing.go index b379b509d..bad22445e 100644 --- a/state/remote/testing.go +++ b/state/remote/testing.go @@ -26,7 +26,7 @@ func TestClient(t *testing.T, c Client) { t.Fatalf("get: %s", err) } if !bytes.Equal(p.Data, data) { - t.Fatalf("bad: %#v", p) + t.Fatalf("expected full state %q\n\ngot: %q", string(p.Data), string(data)) } if err := c.Delete(); err != nil { @@ -38,7 +38,7 @@ func TestClient(t *testing.T, c Client) { t.Fatalf("get: %s", err) } if p != nil { - t.Fatalf("bad: %#v", p) + t.Fatalf("expected empty state, got: %q", string(p.Data)) } } diff --git a/website/docs/backends/types/remote.html.md b/website/docs/backends/types/remote.html.md new file mode 100644 index 000000000..5429c2d62 --- /dev/null +++ b/website/docs/backends/types/remote.html.md @@ -0,0 +1,118 @@ +--- +layout: "backend-types" +page_title: "Backend Type: remote" +sidebar_current: "docs-backends-types-enhanced-remote" +description: |- + Terraform can store the state and run operations remotely, making it easier to version and work with in a team. +--- + +# remote + +**Kind: Enhanced** + +The remote backend stores state and runs operations remotely. In order +use this backend you need a Terraform Enterprise account or have Private +Terraform Enterprise running on-premises. + +### Commands + +Currently the remote backend supports the following Terraform commands: + + 1. fmt + 2. get + 3. init + 4. output + 5. plan + 6. providers + 7. show + 8. taint + 9. untaint + 10. validate + 11. version + 11. workspace + +### Workspaces +To work with remote workspaces we need either a name or a prefix. You will +get a configuration error when neither or both options are configured. + +#### Name +When a name is provided, that name is used to make a one-to-one mapping +between your local “default” workspace and a named remote workspace. This +option assumes you are not using workspaces when working with TF, so it +will act as a backend that does not support names states. + +#### Prefix +When a prefix is provided it will be used to filter and map workspaces that +can be used with a single configuration. This allows you to dynamically +filter and map all remote workspaces with a matching prefix. + +The prefix is added when making calls to the remote backend and stripped +again when receiving the responses. This way any locally used workspace +names will remain the same short names (e.g. “tst”, “acc”) while the remote +names will be mapped by adding the prefix. + +It is assumed that you are only using named workspaces when working with +Terraform and so the “default” workspace is ignored in this case. If there +is a state file for the “default” config, this will give an error during +`terraform init`. If the default workspace is selected when running the +`init` command, the `init` process will succeed but will end with a message +that tells you how to select an existing workspace or create a new one. + +## Example Configuration + +```hcl +terraform { + backend "remote" { + hostname = "app.terraform.io" + organization = "company" + token = "" + + workspaces { + name = "workspace" + prefix = "my-app-" + } + } +} +``` + +We recommend omitting the token which can be provided as an environment +variable or set as [credentials in the CLI Config File](/docs/commands/cli-config.html#credentials). + +## Example Reference + +```hcl +data "terraform_remote_state" "foo" { + backend = "remote" + + config { + organization = "company" + + workspaces { + name = "workspace" + } + } +} +``` + +## Configuration variables + +The following configuration options are supported: + +* `hostname` - (Optional) The remote backend hostname to connect to. Default + to app.terraform.io. +* `organization` - (Required) The name of the organization containing the + targeted workspace(s). +* `token` - (Optional) The token used to authenticate with the remote backend. + If `TFE_TOKEN` is set or credentials for the host are configured in the CLI + Config File, then this this will override any saved value for this. +* `workspaces` - (Required) Workspaces contains arguments used to filter down + to a set of workspaces to work on. Parameters defined below. + +The `workspaces` block supports the following keys: +* `name` - (Optional) A workspace name used to map the default workspace to a + named remote workspace. When configured only the default workspace can be + used. This option conflicts with `prefix`. +* `prefix` - (Optional) A prefix used to filter workspaces using a single + configuration. New workspaces will automatically be prefixed with this + prefix. If omitted only the default workspace can be used. This option + conflicts with `name`. diff --git a/website/docs/backends/types/terraform-enterprise.html.md b/website/docs/backends/types/terraform-enterprise.html.md index ecd87425c..b4312217a 100644 --- a/website/docs/backends/types/terraform-enterprise.html.md +++ b/website/docs/backends/types/terraform-enterprise.html.md @@ -8,6 +8,9 @@ description: |- # terraform enterprise +-> **Deprecated** Please use the new enhanced [remote](/docs/backends/types/remote.html) +backend for storing state and running remote operations in Terraform Enterprise. + **Kind: Standard (with no locking)** Reads and writes state from a [Terraform Enterprise](/docs/enterprise/index.html) diff --git a/website/layouts/backend-types.erb b/website/layouts/backend-types.erb index 072a45baa..78b2187fc 100644 --- a/website/layouts/backend-types.erb +++ b/website/layouts/backend-types.erb @@ -16,6 +16,9 @@ > local + > + remote +