From 35c87836b4dabed54095521cfeb4e0df7f95a6ec Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 11 Mar 2016 11:07:54 -0800 Subject: [PATCH] core: Add terraform_version to state This adds a field terraform_version to the state that represents the Terraform version that wrote that state. If Terraform encounters a state written by a future version, it will error. You must use at least the version that wrote that state. Internally we have fields to override this behavior (StateFutureAllowed), but I chose not to expose them as CLI flags, since the user can just modify the state directly. This is tricky, but should be tricky to represent the horrible disaster that can happen by enabling it. We didn't have to bump the state format version since the absense of the field means it was written by version "0.0.0" which will always be older. In effect though this change will always apply to version 2 of the state since it appears in 0.7 which bumped the version for other purposes. --- command/apply_test.go | 73 +++++++++++++-- command/meta.go | 7 +- command/plan_test.go | 64 +++++++++++++ command/refresh_test.go | 103 +++++++++++++++++++++ helper/resource/testing.go | 10 ++- terraform/context.go | 41 ++++++--- terraform/context_apply_test.go | 10 ++- terraform/context_test.go | 65 +++++++++++++- terraform/plan.go | 2 +- terraform/state.go | 51 ++++++++++- terraform/state_test.go | 153 ++++++++++++++++++++++++++++++++ terraform/version.go | 9 ++ 12 files changed, 554 insertions(+), 34 deletions(-) diff --git a/command/apply_test.go b/command/apply_test.go index dfb4a37e2..01b9fbd8e 100644 --- a/command/apply_test.go +++ b/command/apply_test.go @@ -889,15 +889,6 @@ func TestApply_stateNoExist(t *testing.T) { func TestApply_sensitiveOutput(t *testing.T) { statePath := testTempFile(t) - p := testProvider() - ui := new(cli.MockUi) - c := &ApplyCommand{ - Meta: Meta{ - ContextOpts: testCtxConfig(p), - Ui: ui, - }, - } - args := []string{ "-state", statePath, testFixturePath("apply-sensitive-output"), @@ -916,6 +907,70 @@ func TestApply_sensitiveOutput(t *testing.T) { } } +func TestApply_stateFuture(t *testing.T) { + originalState := testState() + originalState.TFVersion = "99.99.99" + statePath := testStateFile(t, originalState) + + p := testProvider() + ui := new(cli.MockUi) + c := &ApplyCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + args := []string{ + "-state", statePath, + testFixturePath("apply"), + } + if code := c.Run(args); code == 0 { + t.Fatal("should fail") + } + + f, err := os.Open(statePath) + if err != nil { + t.Fatalf("err: %s", err) + } + + newState, err := terraform.ReadState(f) + f.Close() + if err != nil { + t.Fatalf("err: %s", err) + } + + if !newState.Equal(originalState) { + t.Fatalf("bad: %#v", newState) + } + if newState.TFVersion != originalState.TFVersion { + t.Fatalf("bad: %#v", newState) + } +} + +func TestApply_statePast(t *testing.T) { + originalState := testState() + originalState.TFVersion = "0.1.0" + statePath := testStateFile(t, originalState) + + p := testProvider() + ui := new(cli.MockUi) + c := &ApplyCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + args := []string{ + "-state", statePath, + testFixturePath("apply"), + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } +} + func TestApply_vars(t *testing.T) { statePath := testTempFile(t) diff --git a/command/meta.go b/command/meta.go index db4c2a940..092b37dd5 100644 --- a/command/meta.go +++ b/command/meta.go @@ -126,7 +126,8 @@ func (m *Meta) Context(copts contextOpts) (*terraform.Context, bool, error) { "variable values, create a new plan file.") } - return plan.Context(opts), true, nil + ctx, err := plan.Context(opts) + return ctx, true, err } } @@ -158,8 +159,8 @@ func (m *Meta) Context(copts contextOpts) (*terraform.Context, bool, error) { opts.Module = mod opts.Parallelism = copts.Parallelism opts.State = state.State() - ctx := terraform.NewContext(opts) - return ctx, false, nil + ctx, err := terraform.NewContext(opts) + return ctx, false, err } // DataDir returns the directory where local data will be stored. diff --git a/command/plan_test.go b/command/plan_test.go index 9b89018bf..935793174 100644 --- a/command/plan_test.go +++ b/command/plan_test.go @@ -345,6 +345,70 @@ func TestPlan_stateDefault(t *testing.T) { } } +func TestPlan_stateFuture(t *testing.T) { + originalState := testState() + originalState.TFVersion = "99.99.99" + statePath := testStateFile(t, originalState) + + p := testProvider() + ui := new(cli.MockUi) + c := &PlanCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + args := []string{ + "-state", statePath, + testFixturePath("plan"), + } + if code := c.Run(args); code == 0 { + t.Fatal("should fail") + } + + f, err := os.Open(statePath) + if err != nil { + t.Fatalf("err: %s", err) + } + + newState, err := terraform.ReadState(f) + f.Close() + if err != nil { + t.Fatalf("err: %s", err) + } + + if !newState.Equal(originalState) { + t.Fatalf("bad: %#v", newState) + } + if newState.TFVersion != originalState.TFVersion { + t.Fatalf("bad: %#v", newState) + } +} + +func TestPlan_statePast(t *testing.T) { + originalState := testState() + originalState.TFVersion = "0.1.0" + statePath := testStateFile(t, originalState) + + p := testProvider() + ui := new(cli.MockUi) + c := &PlanCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + args := []string{ + "-state", statePath, + testFixturePath("plan"), + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } +} + func TestPlan_vars(t *testing.T) { p := testProvider() ui := new(cli.MockUi) diff --git a/command/refresh_test.go b/command/refresh_test.go index b7cf3b7d1..91ef22b17 100644 --- a/command/refresh_test.go +++ b/command/refresh_test.go @@ -221,6 +221,109 @@ func TestRefresh_defaultState(t *testing.T) { } } +func TestRefresh_futureState(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("err: %s", err) + } + if err := os.Chdir(testFixturePath("refresh")); err != nil { + t.Fatalf("err: %s", err) + } + defer os.Chdir(cwd) + + state := testState() + state.TFVersion = "99.99.99" + statePath := testStateFile(t, state) + + p := testProvider() + ui := new(cli.MockUi) + c := &RefreshCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + args := []string{ + "-state", statePath, + } + if code := c.Run(args); code == 0 { + t.Fatal("should fail") + } + + if p.RefreshCalled { + t.Fatal("refresh should not be called") + } + + f, err := os.Open(statePath) + if err != nil { + t.Fatalf("err: %s", err) + } + + newState, err := terraform.ReadState(f) + f.Close() + if err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(newState.String()) + expected := strings.TrimSpace(state.String()) + if actual != expected { + t.Fatalf("bad:\n\n%s", actual) + } +} + +func TestRefresh_pastState(t *testing.T) { + state := testState() + state.TFVersion = "0.1.0" + statePath := testStateFile(t, state) + + p := testProvider() + ui := new(cli.MockUi) + c := &RefreshCommand{ + Meta: Meta{ + ContextOpts: testCtxConfig(p), + Ui: ui, + }, + } + + p.RefreshFn = nil + p.RefreshReturn = &terraform.InstanceState{ID: "yes"} + + args := []string{ + "-state", statePath, + testFixturePath("refresh"), + } + if code := c.Run(args); code != 0 { + t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String()) + } + + if !p.RefreshCalled { + t.Fatal("refresh should be called") + } + + f, err := os.Open(statePath) + if err != nil { + t.Fatalf("err: %s", err) + } + + newState, err := terraform.ReadState(f) + f.Close() + if err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(newState.String()) + expected := strings.TrimSpace(testRefreshStr) + if actual != expected { + t.Fatalf("bad:\n\n%s", actual) + } + + if newState.TFVersion != terraform.Version { + t.Fatalf("bad:\n\n%s", newState.TFVersion) + } +} + func TestRefresh_outPath(t *testing.T) { state := testState() statePath := testStateFile(t, state) diff --git a/helper/resource/testing.go b/helper/resource/testing.go index 94e03b531..07eec7ae8 100644 --- a/helper/resource/testing.go +++ b/helper/resource/testing.go @@ -284,7 +284,10 @@ func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r // Initialize the context opts.Module = mod opts.State = state - ctx := terraform.NewContext(&opts) + ctx, err := terraform.NewContext(&opts) + if err != nil { + return err + } if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 { if len(es) > 0 { estrs := make([]string, len(es)) @@ -362,7 +365,10 @@ func testStep( opts.Module = mod opts.State = state opts.Destroy = step.Destroy - ctx := terraform.NewContext(&opts) + ctx, err := terraform.NewContext(&opts) + if err != nil { + return state, fmt.Errorf("Error initializing context: %s", err) + } if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 { if len(es) > 0 { estrs := make([]string, len(es)) diff --git a/terraform/context.go b/terraform/context.go index a645f29f7..90947aebf 100644 --- a/terraform/context.go +++ b/terraform/context.go @@ -35,16 +35,17 @@ const ( // ContextOpts are the user-configurable options to create a context with // NewContext. type ContextOpts struct { - Destroy bool - Diff *Diff - Hooks []Hook - Module *module.Tree - Parallelism int - State *State - Providers map[string]ResourceProviderFactory - Provisioners map[string]ResourceProvisionerFactory - Targets []string - Variables map[string]string + Destroy bool + Diff *Diff + Hooks []Hook + Module *module.Tree + Parallelism int + State *State + StateFutureAllowed bool + Providers map[string]ResourceProviderFactory + Provisioners map[string]ResourceProvisionerFactory + Targets []string + Variables map[string]string UIInput UIInput } @@ -78,7 +79,7 @@ type Context struct { // Once a Context is creator, the pointer values within ContextOpts // should not be mutated in any way, since the pointers are copied, not // the values themselves. -func NewContext(opts *ContextOpts) *Context { +func NewContext(opts *ContextOpts) (*Context, error) { // Copy all the hooks and add our stop hook. We don't append directly // to the Config so that we're not modifying that in-place. sh := new(stopHook) @@ -92,6 +93,22 @@ func NewContext(opts *ContextOpts) *Context { state.init() } + // If our state is from the future, then error. Callers can avoid + // this error by explicitly setting `StateFutureAllowed`. + if !opts.StateFutureAllowed && state.FromFutureTerraform() { + return nil, fmt.Errorf( + "Terraform doesn't allow running any operations against a state\n"+ + "that was written by a future Terraform version. The state is\n"+ + "reporting it is written by Terraform '%s'.\n\n"+ + "Please run at least that version of Terraform to continue.", + state.TFVersion) + } + + // Explicitly reset our state version to our current version so that + // any operations we do will write out that our latest version + // has run. + state.TFVersion = Version + // Determine parallelism, default to 10. We do this both to limit // CPU pressure but also to have an extra guard against rate throttling // from providers. @@ -135,7 +152,7 @@ func NewContext(opts *ContextOpts) *Context { parallelSem: NewSemaphore(par), providerInputConfig: make(map[string]map[string]interface{}), sh: sh, - } + }, nil } type ContextGraphOpts struct { diff --git a/terraform/context_apply_test.go b/terraform/context_apply_test.go index c58ee803f..7348dd0dd 100644 --- a/terraform/context_apply_test.go +++ b/terraform/context_apply_test.go @@ -4115,11 +4115,14 @@ func TestContext2Apply_issue5254(t *testing.T) { t.Fatalf("err: %s", err) } - ctx = planFromFile.Context(&ContextOpts{ + ctx, err = planFromFile.Context(&ContextOpts{ Providers: map[string]ResourceProviderFactory{ "template": testProviderFuncFixed(p), }, }) + if err != nil { + t.Fatalf("err: %s", err) + } state, err = ctx.Apply() if err != nil { @@ -4189,12 +4192,15 @@ func TestContext2Apply_targetedWithTaintedInState(t *testing.T) { t.Fatalf("err: %s", err) } - ctx = planFromFile.Context(&ContextOpts{ + ctx, err = planFromFile.Context(&ContextOpts{ Module: testModule(t, "apply-tainted-targets"), Providers: map[string]ResourceProviderFactory{ "aws": testProviderFuncFixed(p), }, }) + if err != nil { + t.Fatalf("err: %s", err) + } state, err := ctx.Apply() if err != nil { diff --git a/terraform/context_test.go b/terraform/context_test.go index 015ae1921..eee648cd2 100644 --- a/terraform/context_test.go +++ b/terraform/context_test.go @@ -7,8 +7,71 @@ import ( "time" ) +func TestNewContextState(t *testing.T) { + cases := map[string]struct { + Input *ContextOpts + Err bool + }{ + "empty TFVersion": { + &ContextOpts{ + State: &State{}, + }, + false, + }, + + "past TFVersion": { + &ContextOpts{ + State: &State{TFVersion: "0.1.2"}, + }, + false, + }, + + "equal TFVersion": { + &ContextOpts{ + State: &State{TFVersion: Version}, + }, + false, + }, + + "future TFVersion": { + &ContextOpts{ + State: &State{TFVersion: "99.99.99"}, + }, + true, + }, + + "future TFVersion, allowed": { + &ContextOpts{ + State: &State{TFVersion: "99.99.99"}, + StateFutureAllowed: true, + }, + false, + }, + } + + for k, tc := range cases { + ctx, err := NewContext(tc.Input) + if (err != nil) != tc.Err { + t.Fatalf("%s: err: %s", k, err) + } + if err != nil { + continue + } + + // Version should always be set to our current + if ctx.state.TFVersion != Version { + t.Fatalf("%s: state not set to current version", k) + } + } +} + func testContext2(t *testing.T, opts *ContextOpts) *Context { - return NewContext(opts) + ctx, err := NewContext(opts) + if err != nil { + t.Fatalf("err: %s", err) + } + + return ctx } func testApplyFn( diff --git a/terraform/plan.go b/terraform/plan.go index b15ea5c59..b2ff008ee 100644 --- a/terraform/plan.go +++ b/terraform/plan.go @@ -34,7 +34,7 @@ type Plan struct { // // The following fields in opts are overridden by the plan: Config, // Diff, State, Variables. -func (p *Plan) Context(opts *ContextOpts) *Context { +func (p *Plan) Context(opts *ContextOpts) (*Context, error) { opts.Diff = p.Diff opts.Module = p.Module opts.State = p.State diff --git a/terraform/state.go b/terraform/state.go index 0a2c9e5bf..0e5a0fe6e 100644 --- a/terraform/state.go +++ b/terraform/state.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" + "github.com/hashicorp/go-version" "github.com/hashicorp/terraform/config" ) @@ -30,6 +31,9 @@ type State struct { // Version is the protocol version. Currently only "1". Version int `json:"version"` + // TFVersion is the version of Terraform that wrote this state. + TFVersion string `json:"terraform_version,omitempty"` + // Serial is incremented on any operation that modifies // the State file. It is used to detect potentially conflicting // updates. @@ -362,9 +366,10 @@ func (s *State) DeepCopy() *State { return nil } n := &State{ - Version: s.Version, - Serial: s.Serial, - Modules: make([]*ModuleState, 0, len(s.Modules)), + Version: s.Version, + TFVersion: s.TFVersion, + Serial: s.Serial, + Modules: make([]*ModuleState, 0, len(s.Modules)), } for _, mod := range s.Modules { n.Modules = append(n.Modules, mod.deepcopy()) @@ -387,7 +392,7 @@ func (s *State) IncrementSerialMaybe(other *State) { if s.Serial > other.Serial { return } - if !s.Equal(other) { + if other.TFVersion != s.TFVersion || !s.Equal(other) { if other.Serial > s.Serial { s.Serial = other.Serial } @@ -396,6 +401,18 @@ func (s *State) IncrementSerialMaybe(other *State) { } } +// FromFutureTerraform checks if this state was written by a Terraform +// version from the future. +func (s *State) FromFutureTerraform() bool { + // No TF version means it is certainly from the past + if s.TFVersion == "" { + return false + } + + v := version.Must(version.NewVersion(s.TFVersion)) + return SemVersion.LessThan(v) +} + func (s *State) init() { if s.Version == 0 { s.Version = StateVersion @@ -1335,6 +1352,19 @@ func ReadState(src io.Reader) (*State, error) { state.Version) } + // Make sure the version is semantic + if state.TFVersion != "" { + if _, err := version.NewVersion(state.TFVersion); err != nil { + return nil, fmt.Errorf( + "State contains invalid version: %s\n\n"+ + "Terraform validates the version format prior to writing it. This\n"+ + "means that this is invalid of the state becoming corrupted through\n"+ + "some external means. Please manually modify the Terraform version\n"+ + "field to be a proper semantic version.", + state.TFVersion) + } + } + // Sort it state.sort() @@ -1349,6 +1379,19 @@ func WriteState(d *State, dst io.Writer) error { // Ensure the version is set d.Version = StateVersion + // If the TFVersion is set, verify it. We used to just set the version + // here, but this isn't safe since it changes the MD5 sum on some remote + // state storage backends such as Atlas. We now leave it be if needed. + if d.TFVersion != "" { + if _, err := version.NewVersion(d.TFVersion); err != nil { + return fmt.Errorf( + "Error writing state, invalid version: %s\n\n"+ + "The Terraform version when writing the state must be a semantic\n"+ + "version.", + d.TFVersion) + } + } + // Encode the data in a human-friendly way data, err := json.MarshalIndent(d, "", " ") if err != nil { diff --git a/terraform/state_test.go b/terraform/state_test.go index b66aa28a2..20b04742b 100644 --- a/terraform/state_test.go +++ b/terraform/state_test.go @@ -175,6 +175,35 @@ func TestStateModuleOrphans_deepNestedNilConfig(t *testing.T) { } } +func TestStateDeepCopy(t *testing.T) { + cases := []struct { + One, Two *State + F func(*State) interface{} + }{ + // Version + { + &State{Version: 5}, + &State{Version: 5}, + func(s *State) interface{} { return s.Version }, + }, + + // TFVersion + { + &State{TFVersion: "5"}, + &State{TFVersion: "5"}, + func(s *State) interface{} { return s.TFVersion }, + }, + } + + for i, tc := range cases { + actual := tc.F(tc.One.DeepCopy()) + expected := tc.F(tc.Two) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("Bad: %d\n\n%s\n\n%s", i, actual, expected) + } + } +} + func TestStateEqual(t *testing.T) { cases := []struct { Result bool @@ -348,6 +377,11 @@ func TestStateIncrementSerialMaybe(t *testing.T) { }, 5, }, + "S2 has a different TFVersion": { + &State{TFVersion: "0.1"}, + &State{TFVersion: "0.2"}, + 1, + }, } for name, tc := range cases { @@ -987,6 +1021,34 @@ func TestStateEmpty(t *testing.T) { } } +func TestStateFromFutureTerraform(t *testing.T) { + cases := []struct { + In string + Result bool + }{ + { + "", + false, + }, + { + "0.1", + false, + }, + { + "999.15.1", + true, + }, + } + + for _, tc := range cases { + state := &State{TFVersion: tc.In} + actual := state.FromFutureTerraform() + if actual != tc.Result { + t.Fatalf("%s: bad: %v", tc.In, actual) + } + } +} + func TestStateIsRemote(t *testing.T) { cases := []struct { In *State @@ -1206,6 +1268,97 @@ func TestReadStateNewVersion(t *testing.T) { } } +func TestReadStateTFVersion(t *testing.T) { + type tfVersion struct { + TFVersion string `json:"terraform_version"` + } + + cases := []struct { + Written string + Read string + Err bool + }{ + { + "0.0.0", + "0.0.0", + false, + }, + { + "", + "", + false, + }, + { + "bad", + "", + true, + }, + } + + for _, tc := range cases { + buf, err := json.Marshal(&tfVersion{tc.Written}) + if err != nil { + t.Fatalf("err: %v", err) + } + + s, err := ReadState(bytes.NewReader(buf)) + if (err != nil) != tc.Err { + t.Fatalf("%s: err: %s", tc.Written, err) + } + if err != nil { + continue + } + + if s.TFVersion != tc.Read { + t.Fatalf("%s: bad: %s", tc.Written, s.TFVersion) + } + } +} + +func TestWriteStateTFVersion(t *testing.T) { + cases := []struct { + Write string + Read string + Err bool + }{ + { + "0.0.0", + "0.0.0", + false, + }, + { + "", + "", + false, + }, + { + "bad", + "", + true, + }, + } + + for _, tc := range cases { + var buf bytes.Buffer + err := WriteState(&State{TFVersion: tc.Write}, &buf) + if (err != nil) != tc.Err { + t.Fatalf("%s: err: %s", tc.Write, err) + } + if err != nil { + continue + } + + s, err := ReadState(&buf) + if err != nil { + t.Fatalf("%s: err: %s", tc.Write, err) + } + + if s.TFVersion != tc.Read { + t.Fatalf("%s: bad: %s", tc.Write, s.TFVersion) + } + } +} + func TestUpgradeV1State(t *testing.T) { old := &StateV1{ Outputs: map[string]string{ diff --git a/terraform/version.go b/terraform/version.go index 9f0ce0b13..e781d9c25 100644 --- a/terraform/version.go +++ b/terraform/version.go @@ -1,5 +1,9 @@ package terraform +import ( + "github.com/hashicorp/go-version" +) + // The main version number that is being run at the moment. const Version = "0.7.0" @@ -7,3 +11,8 @@ const Version = "0.7.0" // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. const VersionPrerelease = "dev" + +// SemVersion is an instance of version.Version. This has the secondary +// benefit of verifying during tests and init time that our version is a +// proper semantic version, which should always be the case. +var SemVersion = version.Must(version.NewVersion(Version))