From 3cfc83e335cbbf4c7b15d8865deaae4eff646ab9 Mon Sep 17 00:00:00 2001 From: Sander van Harmelen Date: Thu, 30 Aug 2018 22:18:43 +0200 Subject: [PATCH] backend/remote: take working directories into account In TFE you can configure a workspace to use a custom working directory. When determining which directory that needs to be uploaded to TFE, this working directory should be taken into account to make sure we are uploading the correct root directory for the workspace. --- backend/remote/backend_mock.go | 152 +++++++++++++----- backend/remote/backend_plan.go | 22 ++- backend/remote/backend_plan_test.go | 37 +++++ .../terraform/main.tf | 1 + .../terraform/output.log | 29 ++++ backend/remote/testing.go | 17 +- 6 files changed, 210 insertions(+), 48 deletions(-) create mode 100644 backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf create mode 100644 backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log diff --git a/backend/remote/backend_mock.go b/backend/remote/backend_mock.go index 9aea17168..e51682eb7 100644 --- a/backend/remote/backend_mock.go +++ b/backend/remote/backend_mock.go @@ -9,21 +9,45 @@ import ( "io" "io/ioutil" "math/rand" + "os" + "path/filepath" tfe "github.com/hashicorp/go-tfe" ) -type mockConfigurationVersions struct { - configVersions map[string]*tfe.ConfigurationVersion - uploadURLs map[string]*tfe.ConfigurationVersion - workspaces map[string]*tfe.ConfigurationVersion +type mockClient struct { + ConfigurationVersions *mockConfigurationVersions + Organizations *mockOrganizations + Plans *mockPlans + Runs *mockRuns + StateVersions *mockStateVersions + Workspaces *mockWorkspaces } -func newMockConfigurationVersions() *mockConfigurationVersions { +func newMockClient() *mockClient { + c := &mockClient{} + c.ConfigurationVersions = newMockConfigurationVersions(c) + c.Organizations = newMockOrganizations(c) + c.Plans = newMockPlans(c) + c.Runs = newMockRuns(c) + c.StateVersions = newMockStateVersions(c) + c.Workspaces = newMockWorkspaces(c) + return c +} + +type mockConfigurationVersions struct { + client *mockClient + configVersions map[string]*tfe.ConfigurationVersion + uploadPaths map[string]string + uploadURLs map[string]*tfe.ConfigurationVersion +} + +func newMockConfigurationVersions(client *mockClient) *mockConfigurationVersions { return &mockConfigurationVersions{ + client: client, configVersions: make(map[string]*tfe.ConfigurationVersion), + uploadPaths: make(map[string]string), uploadURLs: make(map[string]*tfe.ConfigurationVersion), - workspaces: make(map[string]*tfe.ConfigurationVersion), } } @@ -47,7 +71,6 @@ func (m *mockConfigurationVersions) Create(ctx context.Context, workspaceID stri m.configVersions[cv.ID] = cv m.uploadURLs[url] = cv - m.workspaces[workspaceID] = cv return cv, nil } @@ -65,16 +88,19 @@ func (m *mockConfigurationVersions) Upload(ctx context.Context, url, path string if !ok { return errors.New("404 not found") } + m.uploadPaths[cv.ID] = path cv.Status = tfe.ConfigurationUploaded return nil } type mockOrganizations struct { + client *mockClient organizations map[string]*tfe.Organization } -func newMockOrganizations() *mockOrganizations { +func newMockOrganizations(client *mockClient) *mockOrganizations { return &mockOrganizations{ + client: client, organizations: make(map[string]*tfe.Organization), } } @@ -117,32 +143,53 @@ func (m *mockOrganizations) Delete(ctx context.Context, name string) error { } type mockPlans struct { - logs map[string]string - plans map[string]*tfe.Plan + client *mockClient + logs map[string]string + plans map[string]*tfe.Plan } -func newMockPlans() *mockPlans { +func newMockPlans(client *mockClient) *mockPlans { return &mockPlans{ - logs: make(map[string]string), - plans: make(map[string]*tfe.Plan), + client: client, + logs: make(map[string]string), + plans: make(map[string]*tfe.Plan), } } +// create is a helper function to create a mock plan that uses the configured +// working directory to find the logfile. This enables us to test if we are +// using the +func (m *mockPlans) create(cvID, workspaceID string) (*tfe.Plan, error) { + id := generateID("plan-") + url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", id) + + p := &tfe.Plan{ + ID: id, + LogReadURL: url, + Status: tfe.PlanPending, + } + + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + + m.logs[url] = filepath.Join( + m.client.ConfigurationVersions.uploadPaths[cvID], + w.WorkingDirectory, + "output.log", + ) + m.plans[p.ID] = p + + return p, nil +} + func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) { p, ok := m.plans[planID] if !ok { - url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", planID) - - p = &tfe.Plan{ - ID: planID, - LogReadURL: url, - Status: tfe.PlanFinished, - } - - m.logs[url] = "plan/output.log" - m.plans[p.ID] = p + return nil, tfe.ErrResourceNotFound } - + p.Status = tfe.PlanFinished return p, nil } @@ -157,7 +204,11 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error) return nil, tfe.ErrResourceNotFound } - logs, err := ioutil.ReadFile("./test-fixtures/" + logfile) + if _, err := os.Stat(logfile); os.IsNotExist(err) { + return bytes.NewBufferString("logfile does not exist"), nil + } + + logs, err := ioutil.ReadFile(logfile) if err != nil { return nil, err } @@ -166,34 +217,39 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error) } type mockRuns struct { + client *mockClient runs map[string]*tfe.Run workspaces map[string][]*tfe.Run } -func newMockRuns() *mockRuns { +func newMockRuns(client *mockClient) *mockRuns { return &mockRuns{ + client: client, runs: make(map[string]*tfe.Run), workspaces: make(map[string][]*tfe.Run), } } func (m *mockRuns) List(ctx context.Context, workspaceID string, options tfe.RunListOptions) ([]*tfe.Run, error) { + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound + } var rs []*tfe.Run - for _, r := range m.workspaces[workspaceID] { + for _, r := range m.workspaces[w.ID] { rs = append(rs, r) } return rs, nil } func (m *mockRuns) Create(ctx context.Context, options tfe.RunCreateOptions) (*tfe.Run, error) { - id := generateID("run-") - p := &tfe.Plan{ - ID: generateID("plan-"), - Status: tfe.PlanPending, + p, err := m.client.Plans.create(options.ConfigurationVersion.ID, options.Workspace.ID) + if err != nil { + return nil, err } r := &tfe.Run{ - ID: id, + ID: generateID("run-"), Plan: p, Status: tfe.RunPending, } @@ -225,13 +281,15 @@ func (m *mockRuns) Discard(ctx context.Context, runID string, options tfe.RunDis } type mockStateVersions struct { + client *mockClient states map[string][]byte stateVersions map[string]*tfe.StateVersion workspaces map[string][]string } -func newMockStateVersions() *mockStateVersions { +func newMockStateVersions(client *mockClient) *mockStateVersions { return &mockStateVersions{ + client: client, states: make(map[string][]byte), stateVersions: make(map[string]*tfe.StateVersion), workspaces: make(map[string][]string), @@ -277,14 +335,21 @@ func (m *mockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVe } func (m *mockStateVersions) Current(ctx context.Context, workspaceID string) (*tfe.StateVersion, error) { - svs, ok := m.workspaces[workspaceID] + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + + svs, ok := m.workspaces[w.ID] if !ok || len(svs) == 0 { return nil, tfe.ErrResourceNotFound } + sv, ok := m.stateVersions[svs[len(svs)-1]] if !ok { return nil, tfe.ErrResourceNotFound } + return sv, nil } @@ -297,12 +362,14 @@ func (m *mockStateVersions) Download(ctx context.Context, url string) ([]byte, e } type mockWorkspaces struct { + client *mockClient workspaceIDs map[string]*tfe.Workspace workspaceNames map[string]*tfe.Workspace } -func newMockWorkspaces() *mockWorkspaces { +func newMockWorkspaces(client *mockClient) *mockWorkspaces { return &mockWorkspaces{ + client: client, workspaceIDs: make(map[string]*tfe.Workspace), workspaceNames: make(map[string]*tfe.Workspace), } @@ -317,9 +384,8 @@ func (m *mockWorkspaces) List(ctx context.Context, organization string, options } func (m *mockWorkspaces) Create(ctx context.Context, organization string, options tfe.WorkspaceCreateOptions) (*tfe.Workspace, error) { - id := generateID("ws-") w := &tfe.Workspace{ - ID: id, + ID: generateID("ws-"), Name: *options.Name, } m.workspaceIDs[w.ID] = w @@ -340,8 +406,16 @@ func (m *mockWorkspaces) Update(ctx context.Context, organization, workspace str if !ok { return nil, tfe.ErrResourceNotFound } - w.Name = *options.Name - w.TerraformVersion = *options.TerraformVersion + + if options.Name != nil { + w.Name = *options.Name + } + if options.TerraformVersion != nil { + w.TerraformVersion = *options.TerraformVersion + } + if options.WorkingDirectory != nil { + w.WorkingDirectory = *options.WorkingDirectory + } delete(m.workspaceNames, workspace) m.workspaceNames[w.Name] = w diff --git a/backend/remote/backend_plan.go b/backend/remote/backend_plan.go index 392830125..a3a5c06c4 100644 --- a/backend/remote/backend_plan.go +++ b/backend/remote/backend_plan.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" "os" + "path/filepath" "strings" "time" @@ -64,15 +65,32 @@ func (b *Remote) opPlan(stopCtx, cancelCtx context.Context, op *backend.Operatio var configDir string if op.Module != nil && op.Module.Config().Dir != "" { - configDir = op.Module.Config().Dir + // Make sure to take the working directory into account by removing + // the working directory from the current path. This will result in + // a path that points to the expected root of the workspace. + configDir = filepath.Clean(strings.TrimSuffix( + filepath.Clean(op.Module.Config().Dir), + filepath.Clean(w.WorkingDirectory), + )) } else { + // We did a check earlier to make sure we either have a config dir, + // or the plan is run with -destroy. So this else clause will only + // be executed when we are destroying and doesn't need the config. configDir, err = ioutil.TempDir("", "tf") if err != nil { runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( - generalErr, "error creating temp directory", err))) + generalErr, "error creating temporary directory", err))) return } defer os.RemoveAll(configDir) + + // Make sure the configured working directory exists. + err = os.MkdirAll(filepath.Join(configDir, w.WorkingDirectory), 0700) + if err != nil { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error creating temporary working directory", err))) + return + } } err = b.client.ConfigurationVersions.Upload(stopCtx, cv.UploadURL, configDir) diff --git a/backend/remote/backend_plan_test.go b/backend/remote/backend_plan_test.go index cf3405729..c005e2c33 100644 --- a/backend/remote/backend_plan_test.go +++ b/backend/remote/backend_plan_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + tfe "github.com/hashicorp/go-tfe" "github.com/hashicorp/terraform/backend" "github.com/hashicorp/terraform/config/module" "github.com/hashicorp/terraform/terraform" @@ -179,3 +180,39 @@ func TestRemote_planDestroyNoConfig(t *testing.T) { t.Fatalf("unexpected plan error: %v", run.Err) } } + +func TestRemote_planWithWorkingDirectory(t *testing.T) { + b := testBackendDefault(t) + + options := tfe.WorkspaceUpdateOptions{ + WorkingDirectory: tfe.String("terraform"), + } + + // Configure the workspace to use a custom working direcrtory. + _, err := b.client.Workspaces.Update(context.Background(), b.organization, b.workspace, options) + if err != nil { + t.Fatalf("error configuring working directory: %v", err) + } + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan-with-working-directory/terraform") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + + <-run.Done() + if run.Err != nil { + t.Fatalf("error running operation: %v", run.Err) + } + + output := b.CLI.(*cli.MockUi).OutputWriter.String() + if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") { + t.Fatalf("missing plan summery in output: %s", output) + } +} diff --git a/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf b/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf new file mode 100644 index 000000000..3911a2a9b --- /dev/null +++ b/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf @@ -0,0 +1 @@ +resource "null_resource" "foo" {} diff --git a/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log b/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log new file mode 100644 index 000000000..d9fe98082 --- /dev/null +++ b/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log @@ -0,0 +1,29 @@ +Running plan in the remote backend. Output will stream here. Pressing Ctrl-C +will stop streaming the logs, but will not stop the plan running remotely. +To view this plan in a browser, visit: +https://atlas.local/app/demo1/my-app-web/runs/run-cPK6EnfTpqwy6ucU + +Waiting for the plan to start... + +Terraform v0.11.7 + +Configuring remote state backend... +Initializing Terraform configuration... +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + + null_resource.foo + id: + + +Plan: 1 to add, 0 to change, 0 to destroy. diff --git a/backend/remote/testing.go b/backend/remote/testing.go index 253272f76..7df847a50 100644 --- a/backend/remote/testing.go +++ b/backend/remote/testing.go @@ -69,14 +69,17 @@ func testBackend(t *testing.T, c map[string]interface{}) *Remote { // Configure the backend so the client is created. backend.TestBackendConfig(t, b, c) - // Once the client exists, mock the services we use.. + // Get a new mock client. + mc := newMockClient() + + // Replace the services we use with our mock services. b.CLI = cli.NewMockUi() - b.client.ConfigurationVersions = newMockConfigurationVersions() - b.client.Organizations = newMockOrganizations() - b.client.Plans = newMockPlans() - b.client.Runs = newMockRuns() - b.client.StateVersions = newMockStateVersions() - b.client.Workspaces = newMockWorkspaces() + b.client.ConfigurationVersions = mc.ConfigurationVersions + b.client.Organizations = mc.Organizations + b.client.Plans = mc.Plans + b.client.Runs = mc.Runs + b.client.StateVersions = mc.StateVersions + b.client.Workspaces = mc.Workspaces ctx := context.Background()