ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #11973 from hashicorp/jbardin/state-locking 

Update state.Locker
This commit is contained in:
James Bardin 2017-02-15 17:07:22 -05:00 committed by GitHub
parent 7c83819e78 a372e9c54b
commit 6798cd5911
31 changed files with 471 additions and 670 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
backend/local/backend_apply.go
View File

@ -9,6 +9,7 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
@ -35,10 +36,17 @@ func (b *Local) opApply(
return
}
// If we're locking state, unlock when we're done
if op.LockState {
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()

8
backend/local/backend_local.go
View File

@ -8,7 +8,6 @@ import (
"github.com/hashicorp/errwrap"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
@ -29,13 +28,6 @@ func (b *Local) context(op *backend.Operation) (*terraform.Context, state.State,
return nil, nil, errwrap.Wrapf("Error loading state: {{err}}", err)
}
if op.LockState {
err := clistate.Lock(s, op.Type.String(), b.CLI, b.Colorize())
if err != nil {
return nil, nil, errwrap.Wrapf("Error locking state: {{err}}", err)
}
}
if err := s.RefreshState(); err != nil {
return nil, nil, errwrap.Wrapf("Error loading state: {{err}}", err)
}

12
backend/local/backend_plan.go
View File

@ -13,6 +13,7 @@ import (
"github.com/hashicorp/terraform/command/format"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
@ -59,10 +60,17 @@ func (b *Local) opPlan(
return
}
// If we're locking state, unlock when we're done
if op.LockState {
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()

12
backend/local/backend_refresh.go
View File

@ -9,6 +9,7 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/backend"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
)
func (b *Local) opRefresh(
@ -48,10 +49,17 @@ func (b *Local) opRefresh(
return
}
// If we're locking state, unlock when we're done
if op.LockState {
lockInfo := state.NewLockInfo()
lockInfo.Operation = op.Type.String()
lockID, err := clistate.Lock(opState, lockInfo, b.CLI, b.Colorize())
if err != nil {
runningOp.Err = errwrap.Wrapf("Error locking state: {{err}}", err)
return
}
defer func() {
if err := clistate.Unlock(opState, b.CLI, b.Colorize()); err != nil {
if err := clistate.Unlock(opState, lockID, b.CLI, b.Colorize()); err != nil {
runningOp.Err = multierror.Append(runningOp.Err, err)
}
}()

54
backend/remote-state/consul/client.go
View File

@ -4,10 +4,10 @@ import (
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"time"
consulapi "github.com/hashicorp/consul/api"
"github.com/hashicorp/errwrap"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/state/remote"
@ -58,22 +58,14 @@ func (c *RemoteClient) Delete() error {
return err
}
func (c *RemoteClient) putLockInfo(info string) error {
li := &state.LockInfo{
Path: c.Path,
Created: time.Now().UTC(),
Info: info,
}
js, err := json.Marshal(li)
if err != nil {
return err
}
func (c *RemoteClient) putLockInfo(info *state.LockInfo) error {
info.Path = c.Path
info.Created = time.Now().UTC()
kv := c.Client.KV()
_, err = kv.Put(&consulapi.KVPair{
_, err := kv.Put(&consulapi.KVPair{
Key: c.Path + lockInfoSuffix,
Value: js,
Value: info.Marshal(),
}, nil)
return err
@ -92,22 +84,22 @@ func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) {
li := &state.LockInfo{}
err = json.Unmarshal(pair.Value, li)
if err != nil {
return nil, errwrap.Wrapf("error unmarshaling lock info: {{err}}", err)
return nil, fmt.Errorf("error unmarshaling lock info: %s", err)
}
return li, nil
}
func (c *RemoteClient) Lock(info string) error {
func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) {
select {
case <-c.lockCh:
// We had a lock, but lost it.
// Since we typically only call lock once, we shouldn't ever see this.
return errors.New("lost consul lock")
return "", errors.New("lost consul lock")
default:
if c.lockCh != nil {
// we have an active lock already
return nil
return "", nil
}
}
@ -123,37 +115,47 @@ func (c *RemoteClient) Lock(info string) error {
lock, err := c.Client.LockOpts(opts)
if err != nil {
return nil
return "", err
}
c.consulLock = lock
}
lockErr := &state.LockError{}
lockCh, err := c.consulLock.Lock(make(chan struct{}))
if err != nil {
return err
lockErr.Err = err
return "", lockErr
}
if lockCh == nil {
lockInfo, e := c.getLockInfo()
if e != nil {
return e
lockErr.Err = e
return "", lockErr
}
return lockInfo.Err()
lockErr.Info = lockInfo
return "", lockErr
}
c.lockCh = lockCh
err = c.putLockInfo(info)
if err != nil {
err = multierror.Append(err, c.Unlock())
return err
if unlockErr := c.Unlock(info.ID); unlockErr != nil {
err = multierror.Append(err, unlockErr)
}
return nil
return "", err
}
func (c *RemoteClient) Unlock() error {
return info.ID, nil
}
func (c *RemoteClient) Unlock(id string) error {
// this doesn't use the lock id, because the lock is tied to the consul client.
if c.consulLock == nil || c.lockCh == nil {
return nil
}

3
command/apply_destroy_test.go
View File

@ -45,6 +45,7 @@ func TestApply_destroy(t *testing.T) {
testFixturePath("apply"),
}
if code := c.Run(args); code != 0 {
t.Log(ui.OutputWriter.String())
t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
}
@ -139,7 +140,7 @@ func TestApply_destroyLockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

2
command/apply_test.go
View File

@ -87,7 +87,7 @@ func TestApply_lockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

3
command/command_test.go
View File

@ -583,7 +583,8 @@ func testLockState(sourceDir, path string) (func(), error) {
return deferFunc, fmt.Errorf("read from statelocker returned: %s", err)
}
if string(buf[:n]) != "LOCKED" {
output := string(buf[:n])
if !strings.HasPrefix(output, "LOCKID") {
return deferFunc, fmt.Errorf("statelocker wrote: %s", string(buf[:n]))
}
return deferFunc, nil

28
command/meta_backend.go
View File

@ -533,11 +533,14 @@ func (m *Meta) backendFromPlan(opts *BackendOpts) (backend.Backend, error) {
}
// Lock the state if we can
err = clistate.Lock(realMgr, "backend from plan", m.Ui, m.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from plan"
lockID, err := clistate.Lock(realMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(realMgr, m.Ui, m.Colorize())
defer clistate.Unlock(realMgr, lockID, m.Ui, m.Colorize())
if err := realMgr.RefreshState(); err != nil {
return nil, fmt.Errorf("Error reading state: %s", err)
@ -986,11 +989,14 @@ func (m *Meta) backend_C_r_s(
}
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
// Store the metadata in our saved state location
s := sMgr.State()
@ -1091,11 +1097,14 @@ func (m *Meta) backend_C_r_S_changed(
}
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
// Update the backend state
s = sMgr.State()
@ -1249,11 +1258,14 @@ func (m *Meta) backend_C_R_S_unchanged(
}
// Lock the state if we can
err = clistate.Lock(sMgr, "backend from config", m.Ui, m.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "backend from config"
lockID, err := clistate.Lock(sMgr, lockInfo, m.Ui, m.Colorize())
if err != nil {
return nil, fmt.Errorf("Error locking state: %s", err)
}
defer clistate.Unlock(sMgr, m.Ui, m.Colorize())
defer clistate.Unlock(sMgr, lockID, m.Ui, m.Colorize())
// Unset the remote state
s = sMgr.State()

16
command/meta_backend_migrate.go
View File

@ -24,17 +24,25 @@ import (
//
// This will attempt to lock both states for the migration.
func (m *Meta) backendMigrateState(opts *backendMigrateOpts) error {
err := clistate.Lock(opts.One, "migration source state", m.Ui, m.Colorize())
lockInfoOne := state.NewLockInfo()
lockInfoOne.Operation = "migration"
lockInfoOne.Info = "source state"
lockIDOne, err := clistate.Lock(opts.One, lockInfoOne, m.Ui, m.Colorize())
if err != nil {
return fmt.Errorf("Error locking source state: %s", err)
}
defer clistate.Unlock(opts.One, m.Ui, m.Colorize())
defer clistate.Unlock(opts.One, lockIDOne, m.Ui, m.Colorize())
err = clistate.Lock(opts.Two, "migration destination state", m.Ui, m.Colorize())
lockInfoTwo := state.NewLockInfo()
lockInfoTwo.Operation = "migration"
lockInfoTwo.Info = "destination state"
lockIDTwo, err := clistate.Lock(opts.Two, lockInfoTwo, m.Ui, m.Colorize())
if err != nil {
return fmt.Errorf("Error locking destination state: %s", err)
}
defer clistate.Unlock(opts.Two, m.Ui, m.Colorize())
defer clistate.Unlock(opts.Two, lockIDTwo, m.Ui, m.Colorize())
one := opts.One.State()
two := opts.Two.State()

2
command/plan_test.go
View File

@ -72,7 +72,7 @@ func TestPlan_lockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

2
command/refresh_test.go
View File

@ -91,7 +91,7 @@ func TestRefresh_lockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

18
command/state/state.go
View File

@ -49,14 +49,18 @@ that no one else is holding a lock.
// Lock locks the given state and outputs to the user if locking
// is taking longer than the threshold.
func Lock(s state.State, info string, ui cli.Ui, color *colorstring.Colorize) error {
func Lock(s state.State, info *state.LockInfo, ui cli.Ui, color *colorstring.Colorize) (string, error) {
sl, ok := s.(state.Locker)
if !ok {
return nil
return "", nil
}
var lockID string
err := slowmessage.Do(LockThreshold, func() error {
return sl.Lock(info)
id, err := sl.Lock(info)
lockID = id
return err
}, func() {
if ui != nil {
ui.Output(color.Color(LockMessage))
@ -67,18 +71,20 @@ func Lock(s state.State, info string, ui cli.Ui, color *colorstring.Colorize) er
err = errwrap.Wrapf(strings.TrimSpace(LockErrorMessage), err)
}
return err
return lockID, err
}
// Unlock unlocks the given state and outputs to the user if the
// unlock fails what can be done.
func Unlock(s state.State, ui cli.Ui, color *colorstring.Colorize) error {
func Unlock(s state.State, id string, ui cli.Ui, color *colorstring.Colorize) error {
sl, ok := s.(state.Locker)
if !ok {
return nil
}
err := slowmessage.Do(LockThreshold, sl.Unlock, func() {
err := slowmessage.Do(LockThreshold, func() error {
return sl.Unlock(id)
}, func() {
if ui != nil {
ui.Output(color.Color(UnlockMessage))
}

7
command/taint.go
View File

@ -6,6 +6,7 @@ import (
"strings"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
)
@ -73,13 +74,15 @@ func (c *TaintCommand) Run(args []string) int {
}
if c.Meta.stateLock {
err := clistate.Lock(st, "taint", c.Ui, c.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "taint"
lockID, err := clistate.Lock(st, lockInfo, c.Ui, c.Colorize())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
defer clistate.Unlock(st, c.Ui, c.Colorize())
defer clistate.Unlock(st, lockID, c.Ui, c.Colorize())
}
// Get the actual state structure

2
command/taint_test.go
View File

@ -84,7 +84,7 @@ func TestTaint_lockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

10
command/testdata/statelocker.go vendored
View File

@ -23,17 +23,21 @@ func main() {
Path: os.Args[1],
}
err := s.Lock("command test")
info := state.NewLockInfo()
info.Operation = "test"
info.Info = "state locker"
lockID, err := s.Lock(info)
if err != nil {
io.WriteString(os.Stderr, err.Error())
return
}
// signal to the caller that we're locked
io.WriteString(os.Stdout, "LOCKED")
io.WriteString(os.Stdout, "LOCKID "+lockID)
defer func() {
if err := s.Unlock(); err != nil {
if err := s.Unlock(lockID); err != nil {
io.WriteString(os.Stderr, err.Error())
}
}()

20
command/unlock.go
View File

@ -6,6 +6,7 @@ import (
"github.com/hashicorp/terraform/state"
"github.com/hashicorp/terraform/terraform"
"github.com/mitchellh/cli"
)
// UnlockCommand is a cli.Command implementation that manually unlocks
@ -25,9 +26,21 @@ func (c *UnlockCommand) Run(args []string) int {
return 1
}
args = cmdFlags.Args()
if len(args) == 0 {
c.Ui.Error("unlock requires a lock id argument")
return cli.RunResultHelp
}
lockID := args[0]
if len(args) > 1 {
args = args[1:]
}
// assume everything is initialized. The user can manually init if this is
// required.
configPath, err := ModulePath(cmdFlags.Args())
configPath, err := ModulePath(args)
if err != nil {
c.Ui.Error(err.Error())
return 1
@ -92,7 +105,8 @@ func (c *UnlockCommand) Run(args []string) int {
}
}
if err := s.Unlock(); err != nil {
// FIXME: unlock should require the lock ID
if err := s.Unlock(lockID); err != nil {
c.Ui.Error(fmt.Sprintf("Failed to unlock state: %s", err))
return 1
}
@ -103,7 +117,7 @@ func (c *UnlockCommand) Run(args []string) int {
func (c *UnlockCommand) Help() string {
helpText := `
Usage: terraform force-unlock [DIR]
Usage: terraform force-unlock LOCK_ID [DIR]
Manually unlock the state for the defined configuration.

9
command/unlock_test.go
View File

@ -40,7 +40,12 @@ func TestUnlock(t *testing.T) {
},
}
if code := c.Run([]string{"-force"}); code != 0 {
t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
args := []string{
"-force",
"LOCK_ID",
}
if code := c.Run(args); code != 1 {
t.Fatalf("bad: %d\n%s\n%s", code, ui.OutputWriter.String(), ui.ErrorWriter.String())
}
}

7
command/untaint.go
View File

@ -6,6 +6,7 @@ import (
"strings"
clistate "github.com/hashicorp/terraform/command/state"
"github.com/hashicorp/terraform/state"
)
// UntaintCommand is a cli.Command implementation that manually untaints
@ -61,13 +62,15 @@ func (c *UntaintCommand) Run(args []string) int {
}
if c.Meta.stateLock {
err := clistate.Lock(st, "untaint", c.Ui, c.Colorize())
lockInfo := state.NewLockInfo()
lockInfo.Operation = "untaint"
lockID, err := clistate.Lock(st, lockInfo, c.Ui, c.Colorize())
if err != nil {
c.Ui.Error(fmt.Sprintf("Error locking state: %s", err))
return 1
}
defer clistate.Unlock(st, c.Ui, c.Colorize())
defer clistate.Unlock(st, lockID, c.Ui, c.Colorize())
}
// Get the actual state structure

2
command/untaint_test.go
View File

@ -90,7 +90,7 @@ func TestUntaint_lockedState(t *testing.T) {
}
output := ui.ErrorWriter.String()
if !strings.Contains(output, "locked") {
if !strings.Contains(output, "lock") {
t.Fatal("command output does not look like a lock error:", output)
}
}

10
state/backup.go
View File

@ -42,16 +42,16 @@ func (s *BackupState) PersistState() error {
}
// all states get wrapped by BackupState, so it has to be a Locker
func (s *BackupState) Lock(reason string) error {
func (s *BackupState) Lock(info *LockInfo) (string, error) {
if s, ok := s.Real.(Locker); ok {
return s.Lock(reason)
return s.Lock(info)
}
return nil
return "", nil
}
func (s *BackupState) Unlock() error {
func (s *BackupState) Unlock(id string) error {
if s, ok := s.Real.(Locker); ok {
return s.Unlock()
return s.Unlock(id)
}
return nil
}

290
state/cache.go
View File

@ -1,290 +0,0 @@
package state
import (
"fmt"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/terraform"
)
// CacheState is an implementation of the state interfaces that uses
// a StateReadWriter for a local cache.
type CacheState struct {
Cache CacheStateCache
Durable CacheStateDurable
refreshResult CacheRefreshResult
state *terraform.State
}
// Locker implementation.
// Since remote states are wrapped in a CacheState, we need to implement the
// Lock/Unlock methods here to delegate them to the remote client.
func (s *CacheState) Lock(reason string) error {
durable, durableIsLocker := s.Durable.(Locker)
cache, cacheIsLocker := s.Cache.(Locker)
if durableIsLocker {
if err := durable.Lock(reason); err != nil {
return err
}
}
// We try to lock the Cache too, which is usually a local file. This also
// protects against multiple local processes if the remote state doesn't
// support locking.
if cacheIsLocker {
if err := cache.Lock(reason); err != nil {
// try to unlock Durable if this failed
if unlockErr := durable.Unlock(); unlockErr != nil {
err = multierror.Append(err, unlockErr)
}
return err
}
}
return nil
}
// Unlock unlocks both the Durable and Cache states.
func (s *CacheState) Unlock() error {
durable, durableIsLocker := s.Durable.(Locker)
cache, cacheIsLocker := s.Cache.(Locker)
var err error
if durableIsLocker {
if unlockErr := durable.Unlock(); unlockErr != nil {
err = multierror.Append(err, unlockErr)
}
}
if cacheIsLocker {
if unlockErr := cache.Unlock(); unlockErr != nil {
err = multierror.Append(err, unlockErr)
}
}
return err
}
// StateReader impl.
func (s *CacheState) State() *terraform.State {
return s.state.DeepCopy()
}
// WriteState will write and persist the state to the cache.
//
// StateWriter impl.
func (s *CacheState) WriteState(state *terraform.State) error {
if err := s.Cache.WriteState(state); err != nil {
return err
}
s.state = state
return s.Cache.PersistState()
}
// RefreshState will refresh both the cache and the durable states. It
// can return a myriad of errors (defined at the top of this file) depending
// on potential conflicts that can occur while doing this.
//
// If the durable state is newer than the local cache, then the local cache
// will be replaced with the durable.
//
// StateRefresher impl.
func (s *CacheState) RefreshState() error {
// Refresh the durable state
if err := s.Durable.RefreshState(); err != nil {
return err
}
// Refresh the cached state
if err := s.Cache.RefreshState(); err != nil {
return err
}
// Handle the matrix of cases that can happen when comparing these
// two states.
cached := s.Cache.State()
durable := s.Durable.State()
switch {
case cached == nil && durable == nil:
// Initialized
s.refreshResult = CacheRefreshInit
case cached != nil && durable == nil:
// Cache is newer than remote. Not a big deal, user can just
// persist to get correct state.
s.refreshResult = CacheRefreshLocalNewer
case !cached.HasResources() && durable != nil:
// Cache should be updated since the remote is set but cache isn't
//
// If local is empty then we'll treat it as missing so that
// it can be overwritten by an already-existing remote. This
// allows the user to activate remote state for the first time
// against an already-existing remote state.
s.refreshResult = CacheRefreshUpdateLocal
case durable.Serial < cached.Serial:
// Cache is newer than remote. Not a big deal, user can just
// persist to get correct state.
s.refreshResult = CacheRefreshLocalNewer
case durable.Serial > cached.Serial:
// Cache should be updated since the remote is newer
s.refreshResult = CacheRefreshUpdateLocal
case durable.Serial == cached.Serial:
// They're supposedly equal, verify.
if cached.Equal(durable) {
// Hashes are the same, everything is great
s.refreshResult = CacheRefreshNoop
break
}
// This is very bad. This means we have two state files that
// have the same serial but have a different hash. We can't
// reconcile this. The most likely cause is parallel apply
// operations.
s.refreshResult = CacheRefreshConflict
// Return early so we don't updtae the state
return nil
default:
panic("unhandled cache refresh state")
}
if s.refreshResult == CacheRefreshUpdateLocal {
if err := s.Cache.WriteState(durable); err != nil {
s.refreshResult = CacheRefreshNoop
return err
}
if err := s.Cache.PersistState(); err != nil {
s.refreshResult = CacheRefreshNoop
return err
}
cached = durable
}
s.state = cached
return nil
}
// RefreshResult returns the result of the last refresh.
func (s *CacheState) RefreshResult() CacheRefreshResult {
return s.refreshResult
}
// PersistState takes the local cache, assuming it is newer than the remote
// state, and persists it to the durable storage. If you want to challenge the
// assumption that the local state is the latest, call a RefreshState prior
// to this.
//
// StatePersister impl.
func (s *CacheState) PersistState() error {
if err := s.Durable.WriteState(s.state); err != nil {
return err
}
return s.Durable.PersistState()
}
// CacheStateCache is the meta-interface that must be implemented for
// the cache for the CacheState.
type CacheStateCache interface {
StateReader
StateWriter
StatePersister
StateRefresher
}
// CacheStateDurable is the meta-interface that must be implemented for
// the durable storage for CacheState.
type CacheStateDurable interface {
StateReader
StateWriter
StatePersister
StateRefresher
}
// CacheRefreshResult is used to explain the result of the previous
// RefreshState for a CacheState.
type CacheRefreshResult int
const (
// CacheRefreshNoop indicates nothing has happened,
// but that does not indicate an error. Everything is
// just up to date. (Push/Pull)
CacheRefreshNoop CacheRefreshResult = iota
// CacheRefreshInit indicates that there is no local or
// remote state, and that the state was initialized
CacheRefreshInit
// CacheRefreshUpdateLocal indicates the local state
// was updated. (Pull)
CacheRefreshUpdateLocal
// CacheRefreshUpdateRemote indicates the remote state
// was updated. (Push)
CacheRefreshUpdateRemote
// CacheRefreshLocalNewer means the pull was a no-op
// because the local state is newer than that of the
// server. This means a Push should take place. (Pull)
CacheRefreshLocalNewer
// CacheRefreshRemoteNewer means the push was a no-op
// because the remote state is newer than that of the
// local state. This means a Pull should take place.
// (Push)
CacheRefreshRemoteNewer
// CacheRefreshConflict means that the push or pull
// was a no-op because there is a conflict. This means
// there are multiple state definitions at the same
// serial number with different contents. This requires
// an operator to intervene and resolve the conflict.
// Shame on the user for doing concurrent apply.
// (Push/Pull)
CacheRefreshConflict
)
func (sc CacheRefreshResult) String() string {
switch sc {
case CacheRefreshNoop:
return "Local and remote state in sync"
case CacheRefreshInit:
return "Local state initialized"
case CacheRefreshUpdateLocal:
return "Local state updated"
case CacheRefreshUpdateRemote:
return "Remote state updated"
case CacheRefreshLocalNewer:
return "Local state is newer than remote state, push required"
case CacheRefreshRemoteNewer:
return "Remote state is newer than local state, pull required"
case CacheRefreshConflict:
return "Local and remote state conflict, manual resolution required"
default:
return fmt.Sprintf("Unknown state change type: %d", sc)
}
}
// SuccessfulPull is used to clasify the CacheRefreshResult for
// a refresh operation. This is different by operation, but can be used
// to determine a proper exit code.
func (sc CacheRefreshResult) SuccessfulPull() bool {
switch sc {
case CacheRefreshNoop:
return true
case CacheRefreshInit:
return true
case CacheRefreshUpdateLocal:
return true
case CacheRefreshLocalNewer:
return false
case CacheRefreshConflict:
return false
default:
return false
}
}

118
state/cache_test.go
View File

@ -1,118 +0,0 @@
package state
import (
"fmt"
"os"
"reflect"
"testing"
"github.com/hashicorp/terraform/terraform"
)
func TestCacheState(t *testing.T) {
cache := testLocalState(t)
durable := testLocalState(t)
defer os.Remove(cache.Path)
defer os.Remove(durable.Path)
TestState(t, &CacheState{
Cache: cache,
Durable: durable,
})
}
func TestCacheState_persistDurable(t *testing.T) {
cache := testLocalState(t)
durable := testLocalState(t)
defer os.Remove(cache.Path)
defer os.Remove(durable.Path)
cs := &CacheState{
Cache: cache,
Durable: durable,
}
state := cache.State()
state.Modules = nil
if err := cs.WriteState(state); err != nil {
t.Fatalf("err: %s", err)
}
if reflect.DeepEqual(cache.State(), durable.State()) {
t.Fatal("cache and durable should not be the same")
}
if err := cs.PersistState(); err != nil {
t.Fatalf("err: %s", err)
}
if !reflect.DeepEqual(cache.State(), durable.State()) {
t.Fatalf(
"cache and durable should be the same\n\n%#v\n\n%#v",
cache.State(), durable.State())
}
}
func TestCacheState_RefreshState(t *testing.T) {
for i, test := range []struct {
cacheModules []*terraform.ModuleState
expected CacheRefreshResult
}{
{
cacheModules: nil,
expected: CacheRefreshUpdateLocal,
},
{
cacheModules: []*terraform.ModuleState{},
expected: CacheRefreshUpdateLocal,
},
{
cacheModules: []*terraform.ModuleState{
&terraform.ModuleState{
Path: terraform.RootModulePath,
Resources: map[string]*terraform.ResourceState{
"foo.foo": &terraform.ResourceState{
Primary: &terraform.InstanceState{
ID: "ID",
},
},
},
},
},
expected: CacheRefreshLocalNewer,
},
} {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
cache := testLocalState(t)
durable := testLocalState(t)
defer os.Remove(cache.Path)
defer os.Remove(durable.Path)
cs := &CacheState{
Cache: cache,
Durable: durable,
}
state := cache.State()
state.Modules = test.cacheModules
if err := cs.WriteState(state); err != nil {
t.Fatalf("err: %s", err)
}
if err := cs.RefreshState(); err != nil {
t.Fatalf("err: %s", err)
}
if cs.RefreshResult() != test.expected {
t.Fatalf("bad %d: %v", i, cs.RefreshResult())
}
})
}
}
func TestCacheState_impl(t *testing.T) {
var _ StateReader = new(CacheState)
var _ StateWriter = new(CacheState)
var _ StatePersister = new(CacheState)
var _ StateRefresher = new(CacheState)
}

84
state/local.go
View File

@ -10,6 +10,7 @@ import (
"path/filepath"
"time"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/terraform/terraform"
)
@ -24,6 +25,11 @@ type LocalState struct {
// the file handle corresponding to PathOut
stateFileOut *os.File
// While the stateFileOut will correspond to the lock directly,
// store and check the lock ID to maintain a strict state.Locker
// implementation.
lockID string
// created is set to true if stateFileOut didn't exist before we created it.
// This is mostly so we can clean up emtpy files during tests, but doesn't
// hurt to remove file we never wrote to.
@ -34,27 +40,6 @@ type LocalState struct {
written bool
}
// LockInfo stores metadata for locks taken.
type LockInfo struct {
Path string // Path to the state file
Created time.Time // The time the lock was taken
Info string // Extra info passed to State.Lock
}
// Err returns the lock info formatted in an error
func (l *LockInfo) Err() error {
return fmt.Errorf("state locked. path:%q, created:%s, info:%q",
l.Path, l.Created, l.Info)
}
func (l *LockInfo) String() string {
js, err := json.Marshal(l)
if err != nil {
panic(err)
}
return string(js)
}
// SetState will force a specific state in-memory for this local state.
func (s *LocalState) SetState(state *terraform.State) {
s.state = state
@ -155,28 +140,47 @@ func (s *LocalState) RefreshState() error {
}
// Lock implements a local filesystem state.Locker.
func (s *LocalState) Lock(reason string) error {
func (s *LocalState) Lock(info *LockInfo) (string, error) {
if s.stateFileOut == nil {
if err := s.createStateFiles(); err != nil {
return err
return "", err
}
}
if err := s.lock(); err != nil {
if info, err := s.lockInfo(); err == nil {
return info.Err()
info, infoErr := s.lockInfo()
if infoErr != nil {
err = multierror.Append(err, infoErr)
}
return fmt.Errorf("state file %q locked: %s", s.Path, err)
lockErr := &LockError{
Info: info,
Err: err,
}
return s.writeLockInfo(reason)
return "", lockErr
}
func (s *LocalState) Unlock() error {
// we can't be locked if we don't have a file
if s.stateFileOut == nil {
return nil
s.lockID = info.ID
return s.lockID, s.writeLockInfo(info)
}
func (s *LocalState) Unlock(id string) error {
if s.lockID == "" {
return fmt.Errorf("LocalState not locked")
}
if id != s.lockID {
idErr := fmt.Errorf("invalid lock id: %q. current id: %q", id, s.lockID)
info, err := s.lockInfo()
if err != nil {
err = multierror.Append(idErr, err)
}
return &LockError{
Err: idErr,
Info: info,
}
}
os.Remove(s.lockInfoPath())
@ -187,6 +191,7 @@ func (s *LocalState) Unlock() error {
s.stateFileOut.Close()
s.stateFileOut = nil
s.lockID = ""
// clean up the state file if we created it an never wrote to it
stat, err := os.Stat(fileName)
@ -253,21 +258,12 @@ func (s *LocalState) lockInfo() (*LockInfo, error) {
}
// write a new lock info file
func (s *LocalState) writeLockInfo(info string) error {
func (s *LocalState) writeLockInfo(info *LockInfo) error {
path := s.lockInfoPath()
info.Path = s.Path
info.Created = time.Now().UTC()
lockInfo := &LockInfo{
Path: s.Path,
Created: time.Now().UTC(),
Info: info,
}
infoData, err := json.Marshal(lockInfo)
if err != nil {
panic(fmt.Sprintf("could not marshal lock info: %#v", lockInfo))
}
err = ioutil.WriteFile(path, infoData, 0600)
err := ioutil.WriteFile(path, info.Marshal(), 0600)
if err != nil {
return fmt.Errorf("could not write lock info for %q: %s", s.Path, err)
}

25
state/local_test.go
View File

@ -20,14 +20,16 @@ func TestLocalStateLocks(t *testing.T) {
defer os.Remove(s.Path)
// lock first
if err := s.Lock("test"); err != nil {
info := NewLockInfo()
info.Operation = "test"
lockID, err := s.Lock(info)
if err != nil {
t.Fatal(err)
}
out, err := exec.Command("go", "run", "testdata/lockstate.go", s.Path).CombinedOutput()
if err != nil {
t.Fatal("unexpected lock failure", err)
t.Fatal("unexpected lock failure", err, string(out))
}
if string(out) != "lock failed" {
@ -40,26 +42,28 @@ func TestLocalStateLocks(t *testing.T) {
t.Fatal(err)
}
if lockInfo.Info != "test" {
if lockInfo.Operation != "test" {
t.Fatalf("invalid lock info %#v\n", lockInfo)
}
// a noop, since we unlock on exit
if err := s.Unlock(); err != nil {
if err := s.Unlock(lockID); err != nil {
t.Fatal(err)
}
// local locks can re-lock
if err := s.Lock("test"); err != nil {
lockID, err = s.Lock(info)
if err != nil {
t.Fatal(err)
}
// Unlock should be repeatable
if err := s.Unlock(); err != nil {
if err := s.Unlock(lockID); err != nil {
t.Fatal(err)
}
if err := s.Unlock(); err != nil {
t.Fatal(err)
// we should not be able to unlock the same lock twice
if err := s.Unlock(lockID); err == nil {
t.Fatal("unlocking an unlocked state should fail")
}
// make sure lock info is gone
@ -67,7 +71,6 @@ func TestLocalStateLocks(t *testing.T) {
if _, err := os.Stat(lockInfoPath); !os.IsNotExist(err) {
t.Fatal("lock info not removed")
}
}
func TestLocalState_pathOut(t *testing.T) {

74
state/remote/remote_test.go
View File

@ -2,8 +2,6 @@ package remote
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/hashicorp/terraform/state"
@ -61,75 +59,3 @@ func (nilClient) Get() (*Payload, error) { return nil, nil }
func (c nilClient) Put([]byte) error { return nil }
func (c nilClient) Delete() error { return nil }
// ensure that remote state can be properly initialized
func TestRemoteClient_stateInit(t *testing.T) {
localStateFile, err := ioutil.TempFile("", "tf")
if err != nil {
t.Fatal(err)
}
// we need to remove the temp files so we recognize there's no local or
// remote state.
localStateFile.Close()
os.Remove(localStateFile.Name())
defer os.Remove(localStateFile.Name())
remoteStateFile, err := ioutil.TempFile("", "tf")
if err != nil {
t.Fatal(err)
}
remoteStateFile.Close()
os.Remove(remoteStateFile.Name())
defer os.Remove(remoteStateFile.Name())
// Now we need an empty state to initialize the state files.
newState := terraform.NewState()
newState.Remote = &terraform.RemoteState{
Type: "_local",
Config: map[string]string{"path": remoteStateFile.Name()},
}
remoteClient := &FileClient{
Path: remoteStateFile.Name(),
}
cache := &state.CacheState{
Cache: &state.LocalState{
Path: localStateFile.Name(),
},
Durable: &State{
Client: remoteClient,
},
}
// This will write the local state file, and set the state field in the CacheState
err = cache.WriteState(newState)
if err != nil {
t.Fatal(err)
}
// This will persist the local state we just wrote to the remote state file
err = cache.PersistState()
if err != nil {
t.Fatal(err)
}
// now compare the two state files just to be sure
localData, err := ioutil.ReadFile(localStateFile.Name())
if err != nil {
t.Fatal(err)
}
remoteData, err := ioutil.ReadFile(remoteStateFile.Name())
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(localData, remoteData) {
t.Log("state files don't match")
t.Log("Local:\n", string(localData))
t.Log("Remote:\n", string(remoteData))
t.Fatal("failed to initialize remote state")
}
}

72
state/remote/s3.go
View File

@ -8,7 +8,6 @@ import (
"log"
"os"
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@ -17,6 +16,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/go-multierror"
uuid "github.com/hashicorp/go-uuid"
terraformAws "github.com/hashicorp/terraform/builtin/providers/aws"
"github.com/hashicorp/terraform/state"
)
@ -203,22 +203,27 @@ func (c *S3Client) Delete() error {
return err
}
func (c *S3Client) Lock(info string) error {
func (c *S3Client) Lock(info *state.LockInfo) (string, error) {
if c.lockTable == "" {
return nil
return "", nil
}
stateName := fmt.Sprintf("%s/%s", c.bucketName, c.keyName)
lockInfo := &state.LockInfo{
Path: stateName,
Created: time.Now().UTC(),
Info: info,
info.Path = stateName
if info.ID == "" {
lockID, err := uuid.GenerateUUID()
if err != nil {
return "", err
}
info.ID = lockID
}
putParams := &dynamodb.PutItemInput{
Item: map[string]*dynamodb.AttributeValue{
"LockID": {S: aws.String(stateName)},
"Info": {S: aws.String(lockInfo.String())},
"Info": {S: aws.String(string(info.Marshal()))},
},
TableName: aws.String(c.lockTable),
ConditionExpression: aws.String("attribute_not_exists(LockID)"),
@ -226,17 +231,32 @@ func (c *S3Client) Lock(info string) error {
_, err := c.dynClient.PutItem(putParams)
if err != nil {
lockInfo, infoErr := c.getLockInfo()
if infoErr != nil {
err = multierror.Append(err, infoErr)
}
lockErr := &state.LockError{
Err: err,
Info: lockInfo,
}
return "", lockErr
}
return info.ID, nil
}
func (c *S3Client) getLockInfo() (*state.LockInfo, error) {
getParams := &dynamodb.GetItemInput{
Key: map[string]*dynamodb.AttributeValue{
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))},
},
ProjectionExpression: aws.String("LockID, Created, Info"),
ProjectionExpression: aws.String("LockID, Info"),
TableName: aws.String(c.lockTable),
}
resp, err := c.dynClient.GetItem(getParams)
if err != nil {
return fmt.Errorf("s3 state file %q locked, failed to retrieve info: %s", stateName, err)
return nil, err
}
var infoData string
@ -244,32 +264,48 @@ func (c *S3Client) Lock(info string) error {
infoData = *v.S
}
lockInfo = &state.LockInfo{}
lockInfo := &state.LockInfo{}
err = json.Unmarshal([]byte(infoData), lockInfo)
if err != nil {
return fmt.Errorf("s3 state file %q locked, failed get lock info: %s", stateName, err)
return nil, err
}
return lockInfo.Err()
}
return nil
return lockInfo, nil
}
func (c *S3Client) Unlock() error {
func (c *S3Client) Unlock(id string) error {
if c.lockTable == "" {
return nil
}
lockErr := &state.LockError{}
// TODO: store the path and lock ID in separate fields, and have proper
// projection expression only delete the lock if both match, rather than
// checking the ID from the info field first.
lockInfo, err := c.getLockInfo()
if err != nil {
lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err)
return lockErr
}
lockErr.Info = lockInfo
if lockInfo.ID != id {
lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id)
return lockErr
}
params := &dynamodb.DeleteItemInput{
Key: map[string]*dynamodb.AttributeValue{
"LockID": {S: aws.String(fmt.Sprintf("%s/%s", c.bucketName, c.keyName))},
},
TableName: aws.String(c.lockTable),
}
_, err := c.dynClient.DeleteItem(params)
_, err = c.dynClient.DeleteItem(params)
if err != nil {
return err
lockErr.Err = err
return lockErr
}
return nil
}

34
state/remote/testing.go
View File

@ -57,31 +57,41 @@ func TestRemoteLocks(t *testing.T, a, b Client) {
t.Fatal("client B not a state.Locker")
}
if err := lockerA.Lock("test client A"); err != nil {
infoA := state.NewLockInfo()
infoA.Operation = "test"
infoA.Who = "clientA"
infoB := state.NewLockInfo()
infoB.Operation = "test"
infoB.Who = "clientB"
lockIDA, err := lockerA.Lock(infoA)
if err != nil {
t.Fatal("unable to get initial lock:", err)
}
if err := lockerB.Lock("test client B"); err == nil {
lockerA.Unlock()
_, err = lockerB.Lock(infoB)
if err == nil {
lockerA.Unlock(lockIDA)
t.Fatal("client B obtained lock while held by client A")
} else {
t.Log("lock info error:", err)
}
if err := lockerA.Unlock(); err != nil {
if err := lockerA.Unlock(lockIDA); err != nil {
t.Fatal("error unlocking client A", err)
}
if err := lockerB.Lock("test client B"); err != nil {
lockIDB, err := lockerB.Lock(infoB)
if err != nil {
t.Fatal("unable to obtain lock from client B")
}
if err := lockerB.Unlock(); err != nil {
if lockIDB == lockIDA {
t.Fatalf("duplicate lock IDs: %q", lockIDB)
}
if err = lockerB.Unlock(lockIDB); err != nil {
t.Fatal("error unlocking client B:", err)
}
// unlock should be repeatable
if err := lockerA.Unlock(); err != nil {
t.Fatal("Unlock error from client A when state was not locked:", err)
}
// TODO: Should we enforce that Unlock requires the correct ID?
}

137
state/state.go
View File

@ -1,9 +1,27 @@
package state
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"os/user"
"strings"
"text/template"
"time"
uuid "github.com/hashicorp/go-uuid"
"github.com/hashicorp/terraform/terraform"
)
var rngSource *rand.Rand
func init() {
rngSource = rand.New(rand.NewSource(time.Now().UnixNano()))
}
// State is the collection of all state interfaces.
type State interface {
StateReader
@ -42,9 +60,120 @@ type StatePersister interface {
}
// Locker is implemented to lock state during command execution.
// The optional info parameter can be recorded with the lock, but the
// implementation should not depend in its value.
// The info parameter can be recorded with the lock, but the
// implementation should not depend in its value. The string returned by Lock
// is an ID corresponding to the lock acquired, and must be passed to Unlock to
// ensure that the correct lock is being released.
//
// Lock and Unlock may return an error value of type LockError which in turn
// can contain the LockInfo of a conflicting lock.
type Locker interface {
Lock(info string) error
Unlock() error
Lock(info *LockInfo) (string, error)
Unlock(id string) error
}
// Generate a LockInfo structure, populating the required fields.
func NewLockInfo() *LockInfo {
// this doesn't need to be cryptographically secure, just unique.
// Using math/rand alleviates the need to check handle the read error.
// Use a uuid format to match other IDs used throughout Terraform.
buf := make([]byte, 16)
rngSource.Read(buf)
id, err := uuid.FormatUUID(buf)
if err != nil {
// this of course shouldn't happen
panic(err)
}
// don't error out on user and hostname, as we don't require them
userName := ""
if userInfo, err := user.Current(); err == nil {
userName = userInfo.Username
}
host, _ := os.Hostname()
info := &LockInfo{
ID: id,
Who: fmt.Sprintf("%s@%s", userName, host),
Version: terraform.Version,
Created: time.Now().UTC(),
}
return info
}
// LockInfo stores lock metadata.
//
// Only Operation and Info are required to be set by the caller of Lock.
type LockInfo struct {
// Unique ID for the lock. NewLockInfo provides a random ID, but this may
// be overridden by the lock implementation. The final value if ID will be
// returned by the call to Lock.
ID string
// Terraform operation, provided by the caller.
Operation string
// Extra information to store with the lock, provided by the caller.
Info string
// user@hostname when available
Who string
// Terraform version
Version string
// Time that the lock was taken.
Created time.Time
// Path to the state file when applicable. Set by the Lock implementation.
Path string
}
// Err returns the lock info formatted in an error
func (l *LockInfo) Err() error {
return errors.New(l.String())
}
// Marshal returns a string json representation of the LockInfo
func (l *LockInfo) Marshal() []byte {
js, err := json.Marshal(l)
if err != nil {
panic(err)
}
return js
}
// String return a multi-line string representation of LockInfo
func (l *LockInfo) String() string {
tmpl := `Lock Info:
ID: {{.ID}}
Path: {{.Path}}
Operation: {{.Operation}}
Who: {{.Who}}
Version: {{.Version}}
Created: {{.Created}}
Info: {{.Info}}
`
t := template.Must(template.New("LockInfo").Parse(tmpl))
var out bytes.Buffer
if err := t.Execute(&out, l); err != nil {
panic(err)
}
return out.String()
}
type LockError struct {
Info *LockInfo
Err error
}
func (e *LockError) Error() string {
var out []string
if e.Err != nil {
out = append(out, e.Err.Error())
}
if e.Info != nil {
out = append(out, e.Info.String())
}
return strings.Join(out, "\n")
}

29
state/state_test.go
View File

@ -1,6 +1,7 @@
package state
import (
"encoding/json"
"flag"
"io/ioutil"
"log"
@ -21,3 +22,31 @@ func TestMain(m *testing.M) {
}
os.Exit(m.Run())
}
func TestNewLockInfo(t *testing.T) {
info1 := NewLockInfo()
info2 := NewLockInfo()
if info1.ID == "" {
t.Fatal("LockInfo missing ID")
}
if info1.Version == "" {
t.Fatal("LockInfo missing version")
}
if info1.Created.IsZero() {
t.Fatal("LockInfo missing Created")
}
if info1.ID == info2.ID {
t.Fatal("multiple LockInfo with identical IDs")
}
// test the JSON output is valid
newInfo := &LockInfo{}
err := json.Unmarshal(info1.Marshal(), newInfo)
if err != nil {
t.Fatal(err)
}
}

7
state/testdata/lockstate.go vendored
View File

@ -19,10 +19,15 @@ func main() {
Path: os.Args[1],
}
err := s.Lock("test")
info := state.NewLockInfo()
info.Operation = "test"
info.Info = "state locker"
_, err := s.Lock(info)
if err != nil {
io.WriteString(os.Stderr, "lock failed")
}
return
}