terraform/internal/states/remote/state.go

245 lines
6.6 KiB
Go
Raw Normal View History

2015-02-21 20:52:55 +01:00
package remote
import (
"bytes"
"fmt"
2017-05-25 17:01:25 +02:00
"sync"
2015-02-21 20:52:55 +01:00
uuid "github.com/hashicorp/go-uuid"
"github.com/hashicorp/terraform/internal/states"
"github.com/hashicorp/terraform/internal/states/statefile"
"github.com/hashicorp/terraform/internal/states/statemgr"
2015-02-21 20:52:55 +01:00
)
// State implements the State interfaces in the state package to handle
// reading and writing the remote state. This State on its own does no
// local caching so every persist will go to the remote storage and local
// writes will go to memory.
type State struct {
2017-05-25 17:01:25 +02:00
mu sync.Mutex
2015-02-21 20:52:55 +01:00
Client Client
// We track two pieces of meta data in addition to the state itself:
//
// lineage - the state's unique ID
// serial - the monotonic counter of "versions" of the state
//
// Both of these (along with state) have a sister field
// that represents the values read in from an existing source.
// All three of these values are used to determine if the new
// state has changed from an existing state we read in.
lineage, readLineage string
serial, readSerial uint64
state, readState *states.State
disableLocks bool
2015-02-21 20:52:55 +01:00
}
var _ statemgr.Full = (*State)(nil)
var _ statemgr.Migrator = (*State)(nil)
// statemgr.Reader impl.
func (s *State) State() *states.State {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
2015-02-24 06:36:35 +01:00
return s.state.DeepCopy()
2015-02-21 20:52:55 +01:00
}
// StateForMigration is part of our implementation of statemgr.Migrator.
func (s *State) StateForMigration() *statefile.File {
s.mu.Lock()
defer s.mu.Unlock()
return statefile.New(s.state.DeepCopy(), s.lineage, s.serial)
}
// statemgr.Writer impl.
func (s *State) WriteState(state *states.State) error {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
state: more robust handling of state Serial Previously we relied on a constellation of coincidences for everything to work out correctly with state serials. In particular, callers needed to be very careful about mutating states (or not) because many different bits of code shared pointers to the same objects. Here we move to a model where all of the state managers always use distinct instances of state, copied when WriteState is called. This means that they are truly a snapshot of the state as it was at that call, even if the caller goes on mutating the state that was passed. We also adjust the handling of serials so that the state managers ignore any serials in incoming states and instead just treat each Persist as the next version after what was most recently Refreshed. (An exception exists for when nothing has been refreshed, e.g. because we are writing a state to a location for the first time. In that case we _do_ trust the caller, since the given state is either a new state or it's a copy of something we're migrating from elsewhere with its state and lineage intact.) The intent here is to allow the rest of Terraform to not worry about serials and state identity, and instead just treat the state as a mutable structure. We'll just snapshot it occasionally, when WriteState is called, and deal with serials _only_ at persist time. This is intended as a more robust version of #15423, which was a quick hotfix to an issue that resulted from our previous slopping handling of state serials but arguably makes the problem worse by depending on an additional coincidental behavior of the local backend's apply implementation.
2017-07-05 21:34:30 +02:00
// We create a deep copy of the state here, because the caller also has
// a reference to the given object and can potentially go on to mutate
// it after we return, but we want the snapshot at this point in time.
s.state = state.DeepCopy()
2015-02-21 20:52:55 +01:00
return nil
}
// WriteStateForMigration is part of our implementation of statemgr.Migrator.
func (s *State) WriteStateForMigration(f *statefile.File, force bool) error {
s.mu.Lock()
defer s.mu.Unlock()
if !force {
checkFile := statefile.New(s.state, s.lineage, s.serial)
if err := statemgr.CheckValidImport(f, checkFile); err != nil {
return err
}
}
// The remote backend needs to pass the `force` flag through to its client.
// For backends that support such operations, inform the client
// that a force push has been requested
c, isForcePusher := s.Client.(ClientForcePusher)
if force && isForcePusher {
c.EnableForcePush()
}
// We create a deep copy of the state here, because the caller also has
// a reference to the given object and can potentially go on to mutate
// it after we return, but we want the snapshot at this point in time.
s.state = f.State.DeepCopy()
s.lineage = f.Lineage
s.serial = f.Serial
return nil
}
// statemgr.Refresher impl.
2015-02-21 20:52:55 +01:00
func (s *State) RefreshState() error {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
return s.refreshState()
}
2017-05-25 17:01:25 +02:00
// refreshState is the main implementation of RefreshState, but split out so
// that we can make internal calls to it from methods that are already holding
// the s.mu lock.
func (s *State) refreshState() error {
2015-02-21 20:52:55 +01:00
payload, err := s.Client.Get()
if err != nil {
return err
}
// no remote state is OK
if payload == nil {
s.readState = nil
s.lineage = ""
s.serial = 0
return nil
}
stateFile, err := statefile.Read(bytes.NewReader(payload.Data))
if err != nil {
return err
2015-02-21 20:52:55 +01:00
}
s.lineage = stateFile.Lineage
s.serial = stateFile.Serial
s.state = stateFile.State
// Properties from the remote must be separate so we can
// track changes as lineage, serial and/or state are mutated
s.readLineage = stateFile.Lineage
s.readSerial = stateFile.Serial
s.readState = s.state.DeepCopy()
2015-02-21 20:52:55 +01:00
return nil
}
// statemgr.Persister impl.
2015-02-21 20:52:55 +01:00
func (s *State) PersistState() error {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
if s.readState != nil {
lineageUnchanged := s.readLineage != "" && s.lineage == s.readLineage
serialUnchanged := s.readSerial != 0 && s.serial == s.readSerial
stateUnchanged := statefile.StatesMarshalEqual(s.state, s.readState)
if stateUnchanged && lineageUnchanged && serialUnchanged {
// If the state, lineage or serial haven't changed at all then we have nothing to do.
state/remote: Don't persist snapshot for unchanged state Previously we would write to the backend for every call to PersistState, even if nothing changed since the last write, but update the serial only if the state had changed. The Terraform Cloud & Enterprise state storage have a simple safety check that any future write with an already-used lineage and serial must be byte-for-byte identical. StatesMarshalEqual is intended to detect that, but it only actually detects changes the state itself, and not changes to the snapshot metadata. Because we write the current Terraform version into the snapshot metadata during serialization, we'd previously have an issue where if the first state write after upgrading Terraform to a new version happened to change nothing about the state content then we'd write a new snapshot that differed only by Terraform version, and Terraform Cloud/Enterprise would then reject it. The snapshot header is discarded immediately after decoding, so we can't use information from it when deciding whether to increment the serial. The next best thing is to skip sending no-op snapshot updates to the state client in the first place. These writes are unnecessary anyway, and state storage owners have asked us in the past to elide these to avoid generating noise in their version logs, so we'll also finally meet those requests as a nice side-effect of this change. We didn't previously have tests for the full flow of retrieving and then successively updating persisted state snapshots, so this includes a test which covers that logic and includes an assertion that a no-op update does not get written to the state client.
2019-06-20 02:42:09 +02:00
return nil
}
state/remote: Don't persist snapshot for unchanged state Previously we would write to the backend for every call to PersistState, even if nothing changed since the last write, but update the serial only if the state had changed. The Terraform Cloud & Enterprise state storage have a simple safety check that any future write with an already-used lineage and serial must be byte-for-byte identical. StatesMarshalEqual is intended to detect that, but it only actually detects changes the state itself, and not changes to the snapshot metadata. Because we write the current Terraform version into the snapshot metadata during serialization, we'd previously have an issue where if the first state write after upgrading Terraform to a new version happened to change nothing about the state content then we'd write a new snapshot that differed only by Terraform version, and Terraform Cloud/Enterprise would then reject it. The snapshot header is discarded immediately after decoding, so we can't use information from it when deciding whether to increment the serial. The next best thing is to skip sending no-op snapshot updates to the state client in the first place. These writes are unnecessary anyway, and state storage owners have asked us in the past to elide these to avoid generating noise in their version logs, so we'll also finally meet those requests as a nice side-effect of this change. We didn't previously have tests for the full flow of retrieving and then successively updating persisted state snapshots, so this includes a test which covers that logic and includes an assertion that a no-op update does not get written to the state client.
2019-06-20 02:42:09 +02:00
s.serial++
} else {
// We might be writing a new state altogether, but before we do that
// we'll check to make sure there isn't already a snapshot present
// that we ought to be updating.
err := s.refreshState()
if err != nil {
return fmt.Errorf("failed checking for existing remote state: %s", err)
}
if s.lineage == "" { // indicates that no state snapshot is present yet
lineage, err := uuid.GenerateUUID()
if err != nil {
return fmt.Errorf("failed to generate initial lineage: %v", err)
}
s.lineage = lineage
s.serial = 0
}
state: more robust handling of state Serial Previously we relied on a constellation of coincidences for everything to work out correctly with state serials. In particular, callers needed to be very careful about mutating states (or not) because many different bits of code shared pointers to the same objects. Here we move to a model where all of the state managers always use distinct instances of state, copied when WriteState is called. This means that they are truly a snapshot of the state as it was at that call, even if the caller goes on mutating the state that was passed. We also adjust the handling of serials so that the state managers ignore any serials in incoming states and instead just treat each Persist as the next version after what was most recently Refreshed. (An exception exists for when nothing has been refreshed, e.g. because we are writing a state to a location for the first time. In that case we _do_ trust the caller, since the given state is either a new state or it's a copy of something we're migrating from elsewhere with its state and lineage intact.) The intent here is to allow the rest of Terraform to not worry about serials and state identity, and instead just treat the state as a mutable structure. We'll just snapshot it occasionally, when WriteState is called, and deal with serials _only_ at persist time. This is intended as a more robust version of #15423, which was a quick hotfix to an issue that resulted from our previous slopping handling of state serials but arguably makes the problem worse by depending on an additional coincidental behavior of the local backend's apply implementation.
2017-07-05 21:34:30 +02:00
}
f := statefile.New(s.state, s.lineage, s.serial)
2015-02-21 20:52:55 +01:00
var buf bytes.Buffer
err := statefile.Write(f, &buf)
if err != nil {
2015-02-21 20:52:55 +01:00
return err
}
err = s.Client.Put(buf.Bytes())
state: more robust handling of state Serial Previously we relied on a constellation of coincidences for everything to work out correctly with state serials. In particular, callers needed to be very careful about mutating states (or not) because many different bits of code shared pointers to the same objects. Here we move to a model where all of the state managers always use distinct instances of state, copied when WriteState is called. This means that they are truly a snapshot of the state as it was at that call, even if the caller goes on mutating the state that was passed. We also adjust the handling of serials so that the state managers ignore any serials in incoming states and instead just treat each Persist as the next version after what was most recently Refreshed. (An exception exists for when nothing has been refreshed, e.g. because we are writing a state to a location for the first time. In that case we _do_ trust the caller, since the given state is either a new state or it's a copy of something we're migrating from elsewhere with its state and lineage intact.) The intent here is to allow the rest of Terraform to not worry about serials and state identity, and instead just treat the state as a mutable structure. We'll just snapshot it occasionally, when WriteState is called, and deal with serials _only_ at persist time. This is intended as a more robust version of #15423, which was a quick hotfix to an issue that resulted from our previous slopping handling of state serials but arguably makes the problem worse by depending on an additional coincidental behavior of the local backend's apply implementation.
2017-07-05 21:34:30 +02:00
if err != nil {
return err
}
// After we've successfully persisted, what we just wrote is our new
// reference state until someone calls RefreshState again.
// We've potentially overwritten (via force) the state, lineage
// and / or serial (and serial was incremented) so we copy over all
// three fields so everything matches the new state and a subsequent
// operation would correctly detect no changes to the lineage, serial or state.
state: more robust handling of state Serial Previously we relied on a constellation of coincidences for everything to work out correctly with state serials. In particular, callers needed to be very careful about mutating states (or not) because many different bits of code shared pointers to the same objects. Here we move to a model where all of the state managers always use distinct instances of state, copied when WriteState is called. This means that they are truly a snapshot of the state as it was at that call, even if the caller goes on mutating the state that was passed. We also adjust the handling of serials so that the state managers ignore any serials in incoming states and instead just treat each Persist as the next version after what was most recently Refreshed. (An exception exists for when nothing has been refreshed, e.g. because we are writing a state to a location for the first time. In that case we _do_ trust the caller, since the given state is either a new state or it's a copy of something we're migrating from elsewhere with its state and lineage intact.) The intent here is to allow the rest of Terraform to not worry about serials and state identity, and instead just treat the state as a mutable structure. We'll just snapshot it occasionally, when WriteState is called, and deal with serials _only_ at persist time. This is intended as a more robust version of #15423, which was a quick hotfix to an issue that resulted from our previous slopping handling of state serials but arguably makes the problem worse by depending on an additional coincidental behavior of the local backend's apply implementation.
2017-07-05 21:34:30 +02:00
s.readState = s.state.DeepCopy()
s.readLineage = s.lineage
s.readSerial = s.serial
state: more robust handling of state Serial Previously we relied on a constellation of coincidences for everything to work out correctly with state serials. In particular, callers needed to be very careful about mutating states (or not) because many different bits of code shared pointers to the same objects. Here we move to a model where all of the state managers always use distinct instances of state, copied when WriteState is called. This means that they are truly a snapshot of the state as it was at that call, even if the caller goes on mutating the state that was passed. We also adjust the handling of serials so that the state managers ignore any serials in incoming states and instead just treat each Persist as the next version after what was most recently Refreshed. (An exception exists for when nothing has been refreshed, e.g. because we are writing a state to a location for the first time. In that case we _do_ trust the caller, since the given state is either a new state or it's a copy of something we're migrating from elsewhere with its state and lineage intact.) The intent here is to allow the rest of Terraform to not worry about serials and state identity, and instead just treat the state as a mutable structure. We'll just snapshot it occasionally, when WriteState is called, and deal with serials _only_ at persist time. This is intended as a more robust version of #15423, which was a quick hotfix to an issue that resulted from our previous slopping handling of state serials but arguably makes the problem worse by depending on an additional coincidental behavior of the local backend's apply implementation.
2017-07-05 21:34:30 +02:00
return nil
2015-02-21 20:52:55 +01:00
}
// Lock calls the Client's Lock method if it's implemented.
func (s *State) Lock(info *statemgr.LockInfo) (string, error) {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
if s.disableLocks {
return "", nil
}
if c, ok := s.Client.(ClientLocker); ok {
return c.Lock(info)
}
return "", nil
}
// Unlock calls the Client's Unlock method if it's implemented.
func (s *State) Unlock(id string) error {
2017-05-25 17:01:25 +02:00
s.mu.Lock()
defer s.mu.Unlock()
if s.disableLocks {
return nil
}
if c, ok := s.Client.(ClientLocker); ok {
return c.Unlock(id)
}
return nil
}
// DisableLocks turns the Lock and Unlock methods into no-ops. This is intended
// to be called during initialization of a state manager and should not be
// called after any of the statemgr.Full interface methods have been called.
func (s *State) DisableLocks() {
s.disableLocks = true
}
// StateSnapshotMeta returns the metadata from the most recently persisted
// or refreshed persistent state snapshot.
//
// This is an implementation of statemgr.PersistentMeta.
func (s *State) StateSnapshotMeta() statemgr.SnapshotMeta {
return statemgr.SnapshotMeta{
Lineage: s.lineage,
Serial: s.serial,
}
}