core: Introduce state v3 and upgrade process

This commit makes the current Terraform state version 3 (previously 2),
and a migration process as part of reading v2 state. For the most part
this is unnecessary: helper/schema will deal with upgrading state for
providers written with that framework. However, for providers which
implemented the resource model directly, this gives a best-efforts
attempt at lossless upgrade.

The heuristics used to change the count of a map from the .# key to the
.% key are as follows:

    - if the flat map contains any non-numeric keys, we treat it as a
      map
    - if the map is empty it must be computed or optional, so we remove
      it from state

There is a known edge condition: maps with all-numeric keys are
indistinguishable from sets without access to the schema. They will need
manual conversion or may result in spurious diffs.
This commit is contained in:
James Nugent 2016-06-07 21:12:55 +02:00
parent 75ef7ab636
commit 706ccb7dfe
6 changed files with 413 additions and 207 deletions

View File

@ -250,7 +250,7 @@ func (f *fakeAtlas) handler(resp http.ResponseWriter, req *http.Request) {
// loads the state.
var testStateModuleOrderChange = []byte(
`{
"version": 2,
"version": 3,
"serial": 1,
"modules": [
{
@ -289,7 +289,7 @@ var testStateModuleOrderChange = []byte(
var testStateSimple = []byte(
`{
"version": 2,
"version": 3,
"serial": 1,
"modules": [
{

View File

@ -19,7 +19,7 @@ import (
const (
// StateVersion is the current version for our state file
StateVersion = 2
StateVersion = 3
)
// rootModulePath is the path of the root module
@ -1379,24 +1379,32 @@ type jsonStateVersionIdentifier struct {
Version int `json:"version"`
}
// Check if this is a V0 format - the magic bytes at the start of the file
// should be "tfstate" if so. We no longer support upgrading this type of
// state but return an error message explaining to a user how they can
// upgrade via the 0.6.x series.
func testForV0State(buf *bufio.Reader) error {
start, err := buf.Peek(len("tfstate"))
if err != nil {
return fmt.Errorf("Failed to check for magic bytes: %v", err)
}
if string(start) == "tfstate" {
return fmt.Errorf("Terraform 0.7 no longer supports upgrading the binary state\n" +
"format which was used prior to Terraform 0.3. Please upgrade\n" +
"this state file using Terraform 0.6.16 prior to using it with\n" +
"Terraform 0.7.")
}
return nil
}
// ReadState reads a state structure out of a reader in the format that
// was written by WriteState.
func ReadState(src io.Reader) (*State, error) {
buf := bufio.NewReader(src)
// Check if this is a V0 format - the magic bytes at the start of the file
// should be "tfstate" if so. We no longer support upgrading this type of
// state but display an error message explaining to a user how they can
// upgrade via the 0.6.x series.
start, err := buf.Peek(len("tfstate"))
if err != nil {
return nil, fmt.Errorf("Failed to check for magic bytes: %v", err)
}
if string(start) == "tfstate" {
return nil, fmt.Errorf("Terraform 0.7 no longer supports upgrading the binary state\n" +
"format which was used prior to Terraform 0.3. Please upgrade\n" +
"this state file using Terraform 0.6.16 prior to using it with\n" +
"Terraform 0.7.")
if err := testForV0State(buf); err != nil {
return nil, err
}
// If we are JSON we buffer the whole thing in memory so we can read it twice.
@ -1415,17 +1423,39 @@ func ReadState(src io.Reader) (*State, error) {
case 0:
return nil, fmt.Errorf("State version 0 is not supported as JSON.")
case 1:
old, err := ReadStateV1(jsonBytes)
v1State, err := ReadStateV1(jsonBytes)
if err != nil {
return nil, err
}
return old.upgrade()
v2State, err := upgradeStateV1ToV2(v1State)
if err != nil {
return nil, err
}
v3State, err := upgradeStateV2ToV3(v2State)
if err != nil {
return nil, err
}
return v3State, nil
case 2:
state, err := ReadStateV2(jsonBytes)
v2State, err := ReadStateV2(jsonBytes)
if err != nil {
return nil, err
}
return state, nil
v3State, err := upgradeStateV2ToV3(v2State)
if err != nil {
return nil, err
}
return v3State, nil
case 3:
v3State, err := ReadStateV3(jsonBytes)
if err != nil {
return nil, err
}
return v3State, nil
default:
return nil, fmt.Errorf("State version %d not supported, please update.",
versionIdentifier.Version)
@ -1433,20 +1463,52 @@ func ReadState(src io.Reader) (*State, error) {
}
func ReadStateV1(jsonBytes []byte) (*stateV1, error) {
state := &stateV1{}
v1State := &stateV1{}
if err := json.Unmarshal(jsonBytes, v1State); err != nil {
return nil, fmt.Errorf("Decoding state file failed: %v", err)
}
if v1State.Version != 1 {
return nil, fmt.Errorf("Decoded state version did not match the decoder selection: "+
"read %d, expected 1", v1State.Version)
}
return v1State, nil
}
func ReadStateV2(jsonBytes []byte) (*State, error) {
state := &State{}
if err := json.Unmarshal(jsonBytes, state); err != nil {
return nil, fmt.Errorf("Decoding state file failed: %v", err)
}
if state.Version != 1 {
return nil, fmt.Errorf("Decoded state version did not match the decoder selection: "+
"read %d, expected 1", state.Version)
// Check the version, this to ensure we don't read a future
// version that we don't understand
if state.Version > StateVersion {
return nil, fmt.Errorf("State version %d not supported, please update.",
state.Version)
}
// Make sure the version is semantic
if state.TFVersion != "" {
if _, err := version.NewVersion(state.TFVersion); err != nil {
return nil, fmt.Errorf(
"State contains invalid version: %s\n\n"+
"Terraform validates the version format prior to writing it. This\n"+
"means that this is invalid of the state becoming corrupted through\n"+
"some external means. Please manually modify the Terraform version\n"+
"field to be a proper semantic version.",
state.TFVersion)
}
}
// Sort it
state.sort()
return state, nil
}
func ReadStateV2(jsonBytes []byte) (*State, error) {
func ReadStateV3(jsonBytes []byte) (*State, error) {
state := &State{}
if err := json.Unmarshal(jsonBytes, state); err != nil {
return nil, fmt.Errorf("Decoding state file failed: %v", err)

View File

@ -1128,7 +1128,7 @@ func TestInstanceState_MergeDiff_nilDiff(t *testing.T) {
}
}
func TestReadUpgradeStateV1toV2(t *testing.T) {
func TestReadUpgradeStateV1toV3(t *testing.T) {
// ReadState should transparently detect the old version but will upgrade
// it on Write.
actual, err := ReadState(strings.NewReader(testV1State))
@ -1141,7 +1141,7 @@ func TestReadUpgradeStateV1toV2(t *testing.T) {
t.Fatalf("err: %s", err)
}
if actual.Version != 2 {
if actual.Version != 3 {
t.Fatalf("bad: State version not incremented; is %d", actual.Version)
}
@ -1155,7 +1155,7 @@ func TestReadUpgradeStateV1toV2(t *testing.T) {
}
}
func TestReadUpgradeStateV1toV2_outputs(t *testing.T) {
func TestReadUpgradeStateV1toV3_outputs(t *testing.T) {
// ReadState should transparently detect the old version but will upgrade
// it on Write.
actual, err := ReadState(strings.NewReader(testV1StateWithOutputs))
@ -1168,7 +1168,7 @@ func TestReadUpgradeStateV1toV2_outputs(t *testing.T) {
t.Fatalf("err: %s", err)
}
if actual.Version != 2 {
if actual.Version != 3 {
t.Fatalf("bad: State version not incremented; is %d", actual.Version)
}

View File

@ -0,0 +1,178 @@
package terraform
import (
"fmt"
"github.com/mitchellh/copystructure"
)
// upgradeStateV1ToV2 is used to upgrade a V1 state representation
// into a V2 state representation
func upgradeStateV1ToV2(old *stateV1) (*State, error) {
if old == nil {
return nil, nil
}
remote, err := old.Remote.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading State V1: %v", err)
}
modules := make([]*ModuleState, len(old.Modules))
for i, module := range old.Modules {
upgraded, err := module.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading State V1: %v", err)
}
modules[i] = upgraded
}
if len(modules) == 0 {
modules = nil
}
newState := &State{
Version: 2,
Serial: old.Serial,
Remote: remote,
Modules: modules,
}
newState.sort()
return newState, nil
}
func (old *remoteStateV1) upgradeToV2() (*RemoteState, error) {
if old == nil {
return nil, nil
}
config, err := copystructure.Copy(old.Config)
if err != nil {
return nil, fmt.Errorf("Error upgrading RemoteState V1: %v", err)
}
return &RemoteState{
Type: old.Type,
Config: config.(map[string]string),
}, nil
}
func (old *moduleStateV1) upgradeToV2() (*ModuleState, error) {
if old == nil {
return nil, nil
}
path, err := copystructure.Copy(old.Path)
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
// Outputs needs upgrading to use the new structure
outputs := make(map[string]*OutputState)
for key, output := range old.Outputs {
outputs[key] = &OutputState{
Type: "string",
Value: output,
Sensitive: false,
}
}
if len(outputs) == 0 {
outputs = nil
}
resources := make(map[string]*ResourceState)
for key, oldResource := range old.Resources {
upgraded, err := oldResource.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
resources[key] = upgraded
}
if len(resources) == 0 {
resources = nil
}
dependencies, err := copystructure.Copy(old.Dependencies)
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
return &ModuleState{
Path: path.([]string),
Outputs: outputs,
Resources: resources,
Dependencies: dependencies.([]string),
}, nil
}
func (old *resourceStateV1) upgradeToV2() (*ResourceState, error) {
if old == nil {
return nil, nil
}
dependencies, err := copystructure.Copy(old.Dependencies)
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
primary, err := old.Primary.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
deposed := make([]*InstanceState, len(old.Deposed))
for i, v := range old.Deposed {
upgraded, err := v.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
deposed[i] = upgraded
}
if len(deposed) == 0 {
deposed = nil
}
return &ResourceState{
Type: old.Type,
Dependencies: dependencies.([]string),
Primary: primary,
Deposed: deposed,
Provider: old.Provider,
}, nil
}
func (old *instanceStateV1) upgradeToV2() (*InstanceState, error) {
if old == nil {
return nil, nil
}
attributes, err := copystructure.Copy(old.Attributes)
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
ephemeral, err := old.Ephemeral.upgradeToV2()
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
meta, err := copystructure.Copy(old.Meta)
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
return &InstanceState{
ID: old.ID,
Attributes: attributes.(map[string]string),
Ephemeral: *ephemeral,
Meta: meta.(map[string]string),
}, nil
}
func (old *ephemeralStateV1) upgradeToV2() (*EphemeralState, error) {
connInfo, err := copystructure.Copy(old.ConnInfo)
if err != nil {
return nil, fmt.Errorf("Error upgrading EphemeralState V1: %v", err)
}
return &EphemeralState{
ConnInfo: connInfo.(map[string]string),
}, nil
}

View File

@ -0,0 +1,142 @@
package terraform
import (
"fmt"
"log"
"regexp"
"sort"
"strconv"
"strings"
)
// The upgrade process from V2 to V3 state does not affect the structure,
// so we do not need to redeclare all of the structs involved - we just
// take a deep copy of the old structure and assert the version number is
// as we expect.
func upgradeStateV2ToV3(old *State) (*State, error) {
new := old.DeepCopy()
// Ensure the copied version is v2 before attempting to upgrade
if new.Version != 2 {
return nil, fmt.Errorf("Cannot appply v2->v3 state upgrade to " +
"a state which is not version 2.")
}
// Set the new version number
new.Version = 3
// Change the counts for things which look like maps to use the %
// syntax. Remove counts for empty collections - they will be added
// back in later.
for _, module := range new.Modules {
for _, resource := range module.Resources {
// Upgrade Primary
if resource.Primary != nil {
upgradeAttributesV2ToV3(resource.Primary)
}
// Upgrade Deposed
if resource.Deposed != nil {
for _, deposed := range resource.Deposed {
upgradeAttributesV2ToV3(deposed)
}
}
}
}
return new, nil
}
func upgradeAttributesV2ToV3(instanceState *InstanceState) error {
collectionKeyRegexp := regexp.MustCompile(`^(.*\.)#$`)
collectionSubkeyRegexp := regexp.MustCompile(`^([^\.]+)\..*`)
// Identify the key prefix of anything which is a collection
var collectionKeyPrefixes []string
for key := range instanceState.Attributes {
if submatches := collectionKeyRegexp.FindAllStringSubmatch(key, -1); len(submatches) > 0 {
collectionKeyPrefixes = append(collectionKeyPrefixes, submatches[0][1])
}
}
sort.Strings(collectionKeyPrefixes)
log.Printf("[STATE UPGRADE] Detected the following collections in state: %v", collectionKeyPrefixes)
// This could be rolled into fewer loops, but it is somewhat clearer this way, and will not
// run very often.
for _, prefix := range collectionKeyPrefixes {
// First get the actual keys that belong to this prefix
var potentialKeysMatching []string
for key := range instanceState.Attributes {
if strings.HasPrefix(key, prefix) {
potentialKeysMatching = append(potentialKeysMatching, strings.TrimPrefix(key, prefix))
}
}
sort.Strings(potentialKeysMatching)
var actualKeysMatching []string
for _, key := range potentialKeysMatching {
if submatches := collectionSubkeyRegexp.FindAllStringSubmatch(key, -1); len(submatches) > 0 {
actualKeysMatching = append(actualKeysMatching, submatches[0][1])
} else {
if key != "#" {
actualKeysMatching = append(actualKeysMatching, key)
}
}
}
actualKeysMatching = uniqueSortedStrings(actualKeysMatching)
// Now inspect the keys in order to determine whether this is most likely to be
// a map, list or set. There is room for error here, so we log in each case. If
// there is no method of telling, we remove the key from the InstanceState in
// order that it will be recreated. Again, this could be rolled into fewer loops
// but we prefer clarity.
oldCountKey := fmt.Sprintf("%s#", prefix)
// First, detect "obvious" maps - which have non-numeric keys (mostly).
hasNonNumericKeys := false
for _, key := range actualKeysMatching {
if _, err := strconv.Atoi(key); err != nil {
hasNonNumericKeys = true
}
}
if hasNonNumericKeys {
newCountKey := fmt.Sprintf("%s%%", prefix)
instanceState.Attributes[newCountKey] = instanceState.Attributes[oldCountKey]
delete(instanceState.Attributes, oldCountKey)
log.Printf("[STATE UPGRADE] Detected %s as a map. Replaced count = %s",
strings.TrimSuffix(prefix, "."), instanceState.Attributes[newCountKey])
}
// Now detect empty collections and remove them from state.
if len(actualKeysMatching) == 0 {
delete(instanceState.Attributes, oldCountKey)
log.Printf("[STATE UPGRADE] Detected %s as an empty collection. Removed from state.",
strings.TrimSuffix(prefix, "."))
}
}
return nil
}
// uniqueSortedStrings removes duplicates from a slice of strings and returns
// a sorted slice of the unique strings.
func uniqueSortedStrings(input []string) []string {
uniquemap := make(map[string]struct{})
for _, str := range input {
uniquemap[str] = struct{}{}
}
output := make([]string, len(uniquemap))
i := 0
for key := range uniquemap {
output[i] = key
i = i + 1
}
sort.Strings(output)
return output
}

View File

@ -1,17 +1,13 @@
package terraform
import (
"fmt"
"github.com/mitchellh/copystructure"
)
// stateV1 keeps track of a snapshot state-of-the-world that Terraform
// can use to keep track of what real world resources it is actually
// managing.
//
// stateV1 is _only used for the purposes of backwards compatibility
// and is no longer used in Terraform.
//
// For the upgrade process, see state_upgrade_v1_to_v2.go
type stateV1 struct {
// Version is the protocol version. "1" for a StateV1.
Version int `json:"version"`
@ -29,42 +25,6 @@ type stateV1 struct {
Modules []*moduleStateV1 `json:"modules"`
}
// upgrade is used to upgrade a V1 state representation
// into a State (current) representation.
func (old *stateV1) upgrade() (*State, error) {
if old == nil {
return nil, nil
}
remote, err := old.Remote.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading State V1: %v", err)
}
modules := make([]*ModuleState, len(old.Modules))
for i, module := range old.Modules {
upgraded, err := module.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading State V1: %v", err)
}
modules[i] = upgraded
}
if len(modules) == 0 {
modules = nil
}
newState := &State{
Version: old.Version,
Serial: old.Serial,
Remote: remote,
Modules: modules,
}
newState.sort()
return newState, nil
}
type remoteStateV1 struct {
// Type controls the client we use for the remote state
Type string `json:"type"`
@ -74,22 +34,6 @@ type remoteStateV1 struct {
Config map[string]string `json:"config"`
}
func (old *remoteStateV1) upgrade() (*RemoteState, error) {
if old == nil {
return nil, nil
}
config, err := copystructure.Copy(old.Config)
if err != nil {
return nil, fmt.Errorf("Error upgrading RemoteState V1: %v", err)
}
return &RemoteState{
Type: old.Type,
Config: config.(map[string]string),
}, nil
}
type moduleStateV1 struct {
// Path is the import path from the root module. Modules imports are
// always disjoint, so the path represents amodule tree
@ -121,54 +65,6 @@ type moduleStateV1 struct {
Dependencies []string `json:"depends_on,omitempty"`
}
func (old *moduleStateV1) upgrade() (*ModuleState, error) {
if old == nil {
return nil, nil
}
path, err := copystructure.Copy(old.Path)
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
// Outputs needs upgrading to use the new structure
outputs := make(map[string]*OutputState)
for key, output := range old.Outputs {
outputs[key] = &OutputState{
Type: "string",
Value: output,
Sensitive: false,
}
}
if len(outputs) == 0 {
outputs = nil
}
resources := make(map[string]*ResourceState)
for key, oldResource := range old.Resources {
upgraded, err := oldResource.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
resources[key] = upgraded
}
if len(resources) == 0 {
resources = nil
}
dependencies, err := copystructure.Copy(old.Dependencies)
if err != nil {
return nil, fmt.Errorf("Error upgrading ModuleState V1: %v", err)
}
return &ModuleState{
Path: path.([]string),
Outputs: outputs,
Resources: resources,
Dependencies: dependencies.([]string),
}, nil
}
type resourceStateV1 struct {
// This is filled in and managed by Terraform, and is the resource
// type itself such as "mycloud_instance". If a resource provider sets
@ -220,42 +116,6 @@ type resourceStateV1 struct {
Provider string `json:"provider,omitempty"`
}
func (old *resourceStateV1) upgrade() (*ResourceState, error) {
if old == nil {
return nil, nil
}
dependencies, err := copystructure.Copy(old.Dependencies)
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
primary, err := old.Primary.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
deposed := make([]*InstanceState, len(old.Deposed))
for i, v := range old.Deposed {
upgraded, err := v.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading ResourceState V1: %v", err)
}
deposed[i] = upgraded
}
if len(deposed) == 0 {
deposed = nil
}
return &ResourceState{
Type: old.Type,
Dependencies: dependencies.([]string),
Primary: primary,
Deposed: deposed,
Provider: old.Provider,
}, nil
}
type instanceStateV1 struct {
// A unique ID for this resource. This is opaque to Terraform
// and is only meant as a lookup mechanism for the providers.
@ -277,45 +137,9 @@ type instanceStateV1 struct {
Meta map[string]string `json:"meta,omitempty"`
}
func (old *instanceStateV1) upgrade() (*InstanceState, error) {
if old == nil {
return nil, nil
}
attributes, err := copystructure.Copy(old.Attributes)
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
ephemeral, err := old.Ephemeral.upgrade()
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
meta, err := copystructure.Copy(old.Meta)
if err != nil {
return nil, fmt.Errorf("Error upgrading InstanceState V1: %v", err)
}
return &InstanceState{
ID: old.ID,
Attributes: attributes.(map[string]string),
Ephemeral: *ephemeral,
Meta: meta.(map[string]string),
}, nil
}
type ephemeralStateV1 struct {
// ConnInfo is used for the providers to export information which is
// used to connect to the resource for provisioning. For example,
// this could contain SSH or WinRM credentials.
ConnInfo map[string]string `json:"-"`
}
func (old *ephemeralStateV1) upgrade() (*EphemeralState, error) {
connInfo, err := copystructure.Copy(old.ConnInfo)
if err != nil {
return nil, fmt.Errorf("Error upgrading EphemeralState V1: %v", err)
}
return &EphemeralState{
ConnInfo: connInfo.(map[string]string),
}, nil
}