states/statemgr: Migrate, Import, and Export functions

In our recent refactoring of the state manager interfaces we made serial
and lineage management the responsibility of the state managers
themselves, not exposing them at all to most callers, and allowing for
simple state managers that don't implement them at all.

However, we do have some specific cases where we need to preserve these
properly when available, such as migration between backends, and the
"terraform state push" and "terraform state pull" commands.

These new functions and their associated optional interface allow the
logic here to be captured in one place and access via some simple
calls. Separating this from the main interface leaves things simple for
the normal uses of state managers.

Since these functions are mostly just thin wrappers around other
functionality, they are not yet well-tested directly, but will be
indirectly tested through the tests of their callers. A subsequent commit
will add more unit tests here.

state/remote/state.go

@@ -28,6 +28,7 @@ type State struct {
 }
 
 var _ statemgr.Full = (*State)(nil)
+var _ statemgr.Migrator = (*State)(nil)
 
 // statemgr.Reader impl.
 func (s *State) State() *states.State {
@@ -37,6 +38,14 @@ func (s *State) State() *states.State {
 	return s.state.DeepCopy()
 }
 
+// StateForMigration is part of our implementation of statemgr.Migrator.
+func (s *State) StateForMigration() *statefile.File {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	return statefile.New(s.state.DeepCopy(), s.lineage, s.serial)
+}
+
 // statemgr.Writer impl.
 func (s *State) WriteState(state *states.State) error {
 	s.mu.Lock()
@@ -50,6 +59,28 @@ func (s *State) WriteState(state *states.State) error {
 	return nil
 }
 
+// WriteStateForMigration is part of our implementation of statemgr.Migrator.
+func (s *State) WriteStateForMigration(f *statefile.File, force bool) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	checkFile := statefile.New(s.state, s.lineage, s.serial)
+	if !force {
+		if err := statemgr.CheckValidImport(f, checkFile); err != nil {
+			return err
+		}
+	}
+
+	// We create a deep copy of the state here, because the caller also has
+	// a reference to the given object and can potentially go on to mutate
+	// it after we return, but we want the snapshot at this point in time.
+	s.state = f.State.DeepCopy()
+	s.lineage = f.Lineage
+	s.serial = f.Serial
+
+	return nil
+}
+
 // statemgr.Refresher impl.
 func (s *State) RefreshState() error {
 	s.mu.Lock()

states/statemgr/filesystem.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"sync"
@@ -62,6 +63,7 @@ type Filesystem struct {
 var (
 	_ Full           = (*Filesystem)(nil)
 	_ PersistentMeta = (*Filesystem)(nil)
+	_ Migrator       = (*Filesystem)(nil)
 )
 
 // NewFilesystem creates a filesystem-based state manager that reads and writes
@@ -121,9 +123,6 @@ func (s *Filesystem) State() *states.State {
 
 // WriteState is an incorrect implementation of Writer that actually also
 // persists.
-// WriteState for LocalState always persists the state as well.
-//
-// StateWriter impl.
 func (s *Filesystem) WriteState(state *states.State) error {
 	// TODO: this should use a more robust method of writing state, by first
 	// writing to a temp file on the same filesystem, and renaming the file over
@@ -137,7 +136,10 @@ func (s *Filesystem) WriteState(state *states.State) error {
 	}
 
 	defer s.mutex()()
+	return s.writeState(state, nil)
+}
 
+func (s *Filesystem) writeState(state *states.State, meta *SnapshotMeta) error {
 	// We'll try to write our backup first, so we can be sure we've created
 	// it successfully before clobbering the original file it came from.
 	if !s.writtenBackup && s.backupFile != nil && s.backupPath != "" && !statefile.StatesMarshalEqual(state, s.backupFile.State) {
@@ -180,9 +182,15 @@ func (s *Filesystem) WriteState(state *states.State) error {
 		return nil
 	}
 
+	if meta == nil {
 		if s.readFile == nil || !statefile.StatesMarshalEqual(s.file.State, s.readFile.State) {
 			s.file.Serial++
 		}
+	} else {
+		// Force new metadata
+		s.file.Lineage = meta.Lineage
+		s.file.Serial = meta.Serial
+	}
 
 	if err := statefile.Write(s.file, s.stateFileOut); err != nil {
 		return err
@@ -345,6 +353,51 @@ func (s *Filesystem) StateSnapshotMeta() SnapshotMeta {
 	}
 }
 
+// StateForMigration is part of our implementation of Migrator.
+func (s *Filesystem) StateForMigration() *statefile.File {
+	return s.file.DeepCopy()
+}
+
+// WriteStateForMigration is part of our implementation of Migrator.
+func (s *Filesystem) WriteStateForMigration(f *statefile.File, force bool) error {
+	if s.readFile == nil {
+		err := s.RefreshState()
+		if err != nil {
+			return err
+		}
+	}
+	defer s.mutex()()
+
+	if !force {
+		err := CheckValidImport(f, s.readFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if s.readFile != nil {
+		log.Printf(
+			"[TRACE] statemgr.Filesystem: Importing snapshot with lineage %q serial %d over snapshot with lineage %q serial %d at %s",
+			f.Lineage, f.Serial,
+			s.readFile.Lineage, s.readFile.Serial,
+			s.path,
+		)
+	} else {
+		log.Printf(
+			"[TRACE] statemgr.Filesystem: Importing snapshot with lineage %q serial %d as the initial state snapshot at %s",
+			f.Lineage, f.Serial,
+			s.path,
+		)
+	}
+
+	err := s.writeState(f.State, &SnapshotMeta{Lineage: f.Lineage, Serial: f.Serial})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Open the state file, creating the directories and file as needed.
 func (s *Filesystem) createStateFiles() error {
 

states/statemgr/migrate.go

@@ -0,0 +1,212 @@
+package statemgr
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform/states/statefile"
+)
+
+// Migrator is an optional interface implemented by state managers that
+// are capable of direct migration of state snapshots with their associated
+// metadata unchanged.
+//
+// This interface is used when available by function Migrate. See that
+// function for more information on how it is used.
+type Migrator interface {
+	PersistentMeta
+
+	// StateForMigration returns a full statefile representing the latest
+	// snapshot (as would be returned by Reader.State) and the associated
+	// snapshot metadata (as would be returned by
+	// PersistentMeta.StateSnapshotMeta).
+	//
+	// Just as with Reader.State, this must not fail.
+	StateForMigration() *statefile.File
+
+	// WriteStateForMigration accepts a full statefile including associated
+	// snapshot metadata, and atomically updates the stored file (as with
+	// Writer.WriteState) and the metadata.
+	//
+	// If "force" is not set, the manager must call CheckValidImport with
+	// the given file and the current file and complete the update only if
+	// that function returns nil. If force is set this may override such
+	// checks, but some backends do not support forcing and so will act
+	// as if force is always true.
+	WriteStateForMigration(f *statefile.File, force bool) error
+}
+
+// Migrate writes the latest transient state snapshot from src into dest,
+// preserving snapshot metadata (serial and lineage) where possible.
+//
+// If both managers implement the optional interface Migrator then it will
+// be used to copy the snapshot and its associated metadata. Otherwise,
+// the normal Reader and Writer interfaces will be used instead.
+//
+// If the destination manager refuses the new state or fails to write it then
+// its error is returned directly.
+//
+// For state managers that also implement Persistent, it is the caller's
+// responsibility to persist the newly-written state after a successful result,
+// just as with calls to Writer.WriteState.
+//
+// This function doesn't do any locking of its own, so if the state managers
+// also implement Locker the caller should hold a lock on both managers
+// for the duration of this call.
+func Migrate(dst, src Transient) error {
+	if dstM, ok := dst.(Migrator); ok {
+		if srcM, ok := src.(Migrator); ok {
+			// Full-fidelity migration, them.
+			s := srcM.StateForMigration()
+			return dstM.WriteStateForMigration(s, true)
+		}
+	}
+
+	// Managers to not support full-fidelity migration, so migration will not
+	// preserve serial/lineage.
+	s := src.State()
+	return dst.WriteState(s)
+}
+
+// Import loads the given state snapshot into the given manager, preserving
+// its metadata (serial and lineage) if the target manager supports metadata.
+//
+// A state manager must implement the optional interface Migrator to get
+// access to the full metadata.
+//
+// Unless "force" is true, Import will check first that the metadata given
+// in the file matches the current snapshot metadata for the manager, if the
+// manager supports metadata. Some managers do not support forcing, so a
+// write with an unsuitable lineage or serial may still be rejected even if
+// "force" is set. "force" has no effect for managers that do not support
+// snapshot metadata.
+//
+// For state managers that also implement Persistent, it is the caller's
+// responsibility to persist the newly-written state after a successful result,
+// just as with calls to Writer.WriteState.
+//
+// This function doesn't do any locking of its own, so if the state manager
+// also implements Locker the caller should hold a lock on it for the
+// duration of this call.
+func Import(f *statefile.File, mgr Transient, force bool) error {
+	if mgrM, ok := mgr.(Migrator); ok {
+		return mgrM.WriteStateForMigration(f, force)
+	}
+
+	// For managers that don't implement Migrator, this is just a normal write
+	// of the state contained in the given file.
+	return mgr.WriteState(f.State)
+}
+
+// Export retrieves the latest state snapshot from the given manager, including
+// its metadata (serial and lineage) where possible.
+//
+// A state manager must also implement either Migrator or PersistentMeta
+// for the metadata to be included. Otherwise, the relevant fields will have
+// zero value in the returned object.
+//
+// For state managers that also implement Persistent, it is the caller's
+// responsibility to refresh from persistent storage first if needed.
+//
+// This function doesn't do any locking of its own, so if the state manager
+// also implements Locker the caller should hold a lock on it for the
+// duration of this call.
+func Export(mgr Reader) *statefile.File {
+	switch mgrT := mgr.(type) {
+	case Migrator:
+		return mgrT.StateForMigration()
+	case PersistentMeta:
+		s := mgr.State()
+		meta := mgrT.StateSnapshotMeta()
+		return statefile.New(s, meta.Lineage, meta.Serial)
+	default:
+		s := mgr.State()
+		return statefile.New(s, "", 0)
+	}
+}
+
+// SnapshotMetaRel describes a relationship between two SnapshotMeta values,
+// returned from the SnapshotMeta.Compare method where the "first" value
+// is the receiver of that method and the "second" is the given argument.
+type SnapshotMetaRel rune
+
+//go:generate stringer -type=SnapshotMetaRel
+
+const (
+	// SnapshotOlder indicates that two snapshots have a common lineage and
+	// that the first has a lower serial value.
+	SnapshotOlder SnapshotMetaRel = '<'
+
+	// SnapshotNewer indicates that two snapshots have a common lineage and
+	// that the first has a higher serial value.
+	SnapshotNewer SnapshotMetaRel = '>'
+
+	// SnapshotEqual indicates that two snapshots have a common lineage and
+	// the same serial value.
+	SnapshotEqual SnapshotMetaRel = '='
+
+	// SnapshotUnrelated indicates that two snapshots have different lineage
+	// and thus cannot be meaningfully compared.
+	SnapshotUnrelated SnapshotMetaRel = '!'
+
+	// SnapshotLegacy indicates that one or both of the snapshots
+	// does not have a lineage at all, and thus no comparison is possible.
+	SnapshotLegacy SnapshotMetaRel = '?'
+)
+
+// Compare determines the relationship, if any, between the given existing
+// SnapshotMeta and the potential "new" SnapshotMeta that is the receiver.
+func (m SnapshotMeta) Compare(existing SnapshotMeta) SnapshotMetaRel {
+	switch {
+	case m.Lineage == "" || existing.Lineage == "":
+		return SnapshotLegacy
+	case m.Lineage != existing.Lineage:
+		return SnapshotUnrelated
+	case m.Serial > existing.Serial:
+		return SnapshotNewer
+	case m.Serial < existing.Serial:
+		return SnapshotOlder
+	default:
+		// both serials are equal, by elimination
+		return SnapshotEqual
+	}
+}
+
+// CheckValidImport returns nil if the "new" snapshot can be imported as a
+// successor of the "existing" snapshot without forcing.
+//
+// If not, an error is returned describing why.
+func CheckValidImport(newFile, existingFile *statefile.File) error {
+	if existingFile == nil || existingFile.State.Empty() {
+		// It's always okay to overwrite an empty state, regardless of
+		// its lineage/serial.
+		return nil
+	}
+	new := SnapshotMeta{
+		Lineage: newFile.Lineage,
+		Serial:  newFile.Serial,
+	}
+	existing := SnapshotMeta{
+		Lineage: existingFile.Lineage,
+		Serial:  existingFile.Serial,
+	}
+	rel := new.Compare(existing)
+	switch rel {
+	case SnapshotNewer:
+		return nil // a newer snapshot is fine
+	case SnapshotLegacy:
+		return nil // anything goes for a legacy state
+	case SnapshotUnrelated:
+		return fmt.Errorf("cannot import state with lineage %q over unrelated state with lineage %q", new.Lineage, existing.Lineage)
+	case SnapshotEqual:
+		if statefile.StatesMarshalEqual(newFile.State, existingFile.State) {
+			// If lineage, serial, and state all match then this is fine.
+			return nil
+		}
+		return fmt.Errorf("cannot overwrite existing state with serial %d with a different state that has the same serial", new.Serial)
+	case SnapshotOlder:
+		return fmt.Errorf("cannot import state with serial %d over newer state with serial %d", new.Serial, existing.Serial)
+	default:
+		// Should never happen, but we'll check to make sure for safety
+		return fmt.Errorf("unsupported state snapshot relationship %s", rel)
+	}
+}

states/statemgr/migrate_test.go

@@ -0,0 +1,102 @@
+package statemgr
+
+import (
+	"testing"
+
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/states"
+	"github.com/hashicorp/terraform/states/statefile"
+)
+
+func TestCheckValidImport(t *testing.T) {
+	barState := states.BuildState(func(s *states.SyncState) {
+		s.SetOutputValue(
+			addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance),
+			cty.StringVal("bar"), false,
+		)
+	})
+	notBarState := states.BuildState(func(s *states.SyncState) {
+		s.SetOutputValue(
+			addrs.OutputValue{Name: "foo"}.Absolute(addrs.RootModuleInstance),
+			cty.StringVal("not bar"), false,
+		)
+	})
+	emptyState := states.NewState()
+
+	tests := map[string]struct {
+		New      *statefile.File
+		Existing *statefile.File
+		WantErr  string
+	}{
+		"exact match": {
+			New:      statefile.New(barState, "lineage", 1),
+			Existing: statefile.New(barState, "lineage", 1),
+			WantErr:  ``,
+		},
+		"overwrite unrelated empty state": {
+			New:      statefile.New(barState, "lineage1", 1),
+			Existing: statefile.New(emptyState, "lineage2", 1),
+			WantErr:  ``,
+		},
+		"different state with same serial": {
+			New:      statefile.New(barState, "lineage", 1),
+			Existing: statefile.New(notBarState, "lineage", 1),
+			WantErr:  `cannot overwrite existing state with serial 1 with a different state that has the same serial`,
+		},
+		"different state with newer serial": {
+			New:      statefile.New(barState, "lineage", 2),
+			Existing: statefile.New(notBarState, "lineage", 1),
+			WantErr:  ``,
+		},
+		"different state with older serial": {
+			New:      statefile.New(barState, "lineage", 1),
+			Existing: statefile.New(notBarState, "lineage", 2),
+			WantErr:  `cannot import state with serial 1 over newer state with serial 2`,
+		},
+		"different lineage with same serial": {
+			New:      statefile.New(barState, "lineage1", 2),
+			Existing: statefile.New(notBarState, "lineage2", 2),
+			WantErr:  `cannot import state with lineage "lineage1" over unrelated state with lineage "lineage2"`,
+		},
+		"different lineage with different serial": {
+			New:      statefile.New(barState, "lineage1", 3),
+			Existing: statefile.New(notBarState, "lineage2", 2),
+			WantErr:  `cannot import state with lineage "lineage1" over unrelated state with lineage "lineage2"`,
+		},
+		"new state is legacy": {
+			New:      statefile.New(barState, "", 2),
+			Existing: statefile.New(notBarState, "lineage", 2),
+			WantErr:  ``,
+		},
+		"old state is legacy": {
+			New:      statefile.New(barState, "lineage", 2),
+			Existing: statefile.New(notBarState, "", 2),
+			WantErr:  ``,
+		},
+		"both states are legacy": {
+			New:      statefile.New(barState, "", 2),
+			Existing: statefile.New(notBarState, "", 2),
+			WantErr:  ``,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			gotErr := CheckValidImport(test.New, test.Existing)
+
+			if test.WantErr == "" {
+				if gotErr != nil {
+					t.Errorf("unexpected error: %s", gotErr)
+				}
+			} else {
+				if gotErr == nil {
+					t.Errorf("succeeded, but want error: %s", test.WantErr)
+				} else if got, want := gotErr.Error(), test.WantErr; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+			}
+		})
+	}
+}

states/statemgr/snapshotmetarel_string.go

@@ -0,0 +1,26 @@
+// Code generated by "stringer -type=SnapshotMetaRel"; DO NOT EDIT.
+
+package statemgr
+
+import "strconv"
+
+const (
+	_SnapshotMetaRel_name_0 = "SnapshotUnrelated"
+	_SnapshotMetaRel_name_1 = "SnapshotOlderSnapshotEqualSnapshotNewerSnapshotLegacy"
+)
+
+var (
+	_SnapshotMetaRel_index_1 = [...]uint8{0, 13, 26, 39, 53}
+)
+
+func (i SnapshotMetaRel) String() string {
+	switch {
+	case i == 33:
+		return _SnapshotMetaRel_name_0
+	case 60 <= i && i <= 63:
+		i -= 60
+		return _SnapshotMetaRel_name_1[_SnapshotMetaRel_index_1[i]:_SnapshotMetaRel_index_1[i+1]]
+	default:
+		return "SnapshotMetaRel(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+}