terraform/states/statefile/version3_upgrade.go

package statefile

import (
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/hashicorp/hcl/v2/hclsyntax"
	"github.com/zclconf/go-cty/cty"
	ctyjson "github.com/zclconf/go-cty/cty/json"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/configs"
	"github.com/hashicorp/terraform/states"
	"github.com/hashicorp/terraform/tfdiags"
)

func upgradeStateV3ToV4(old *stateV3) (*stateV4, error) {

	if old.Serial < 0 {
		// The new format is using uint64 here, which should be fine for any
		// real state (we only used positive integers in practice) but we'll
		// catch this explicitly here to avoid weird behavior if a state file
		// has been tampered with in some way.
		return nil, fmt.Errorf("state has serial less than zero, which is invalid")
	}

	new := &stateV4{
		TerraformVersion: old.TFVersion,
		Serial:           uint64(old.Serial),
		Lineage:          old.Lineage,
		RootOutputs:      map[string]outputStateV4{},
		Resources:        []resourceStateV4{},
	}

	if new.TerraformVersion == "" {
		// Older formats considered this to be optional, but now it's required
		// and so we'll stub it out with something that's definitely older
		// than the version that really created this state.
		new.TerraformVersion = "0.0.0"
	}

	for _, msOld := range old.Modules {
		if len(msOld.Path) < 1 || msOld.Path[0] != "root" {
			return nil, fmt.Errorf("state contains invalid module path %#v", msOld.Path)
		}

		// Convert legacy-style module address into our newer address type.
		// Since these old formats are only generated by versions of Terraform
		// that don't support count and for_each on modules, we can just assume
		// all of the modules are unkeyed.
		moduleAddr := make(addrs.ModuleInstance, len(msOld.Path)-1)
		for i, name := range msOld.Path[1:] {
			if !hclsyntax.ValidIdentifier(name) {
				// If we don't fail here then we'll produce an invalid state
				// version 4 which subsequent operations will reject, so we'll
				// fail early here for safety to make sure we can never
				// inadvertently commit an invalid snapshot to a backend.
				return nil, fmt.Errorf("state contains invalid module path %#v: %q is not a valid identifier; rename it in Terraform 0.11 before upgrading to Terraform 0.12", msOld.Path, name)
			}
			moduleAddr[i] = addrs.ModuleInstanceStep{
				Name:        name,
				InstanceKey: addrs.NoKey,
			}
		}

		// In a v3 state file, a "resource state" is actually an instance
		// state, so we need to fill in a missing level of heirarchy here
		// by lazily creating resource states as we encounter them.
		// We'll track them in here, keyed on the string representation of
		// the resource address.
		resourceStates := map[string]*resourceStateV4{}

		for legacyAddr, rsOld := range msOld.Resources {
			instAddr, err := parseLegacyResourceAddress(legacyAddr)
			if err != nil {
				return nil, err
			}

			resAddr := instAddr.Resource
			rs, exists := resourceStates[resAddr.String()]
			if !exists {
				var modeStr string
				switch resAddr.Mode {
				case addrs.ManagedResourceMode:
					modeStr = "managed"
				case addrs.DataResourceMode:
					modeStr = "data"
				default:
					return nil, fmt.Errorf("state contains resource %s with an unsupported resource mode %#v", resAddr, resAddr.Mode)
				}

				// In state versions prior to 4 we allowed each instance of a
				// resource to have its own provider configuration address,
				// which makes no real sense in practice because providers
				// are associated with resources in the configuration. We
				// elevate that to the resource level during this upgrade,
				// implicitly taking the provider address of the first instance
				// we encounter for each resource. While this is lossy in
				// theory, in practice there is no reason for these values to
				// differ between instances.
				var providerAddr addrs.AbsProviderConfig
				oldProviderAddr := rsOld.Provider
				if strings.Contains(oldProviderAddr, "provider.") {
					// Smells like a new-style provider address, but we'll test it.
					var diags tfdiags.Diagnostics
					providerAddr, diags = addrs.ParseLegacyAbsProviderConfigStr(oldProviderAddr)
					if diags.HasErrors() {
						if strings.Contains(oldProviderAddr, "${") {
							// There seems to be a common misconception that
							// interpolation was valid in provider aliases
							// in 0.11, so we'll use a specialized error
							// message for that case.
							return nil, fmt.Errorf("invalid provider config reference %q for %s: this alias seems to contain a template interpolation sequence, which was not supported but also not error-checked in Terraform 0.11. To proceed, rename the associated provider alias to a valid identifier and apply the change with Terraform 0.11 before upgrading to Terraform 0.12", oldProviderAddr, instAddr)
						}
						return nil, fmt.Errorf("invalid provider config reference %q for %s: %s", oldProviderAddr, instAddr, diags.Err())
					}
				} else {
					// Smells like an old-style module-local provider address,
					// which we'll need to migrate. We'll assume it's referring
					// to the same module the resource is in, which might be
					// incorrect but it'll get fixed up next time any updates
					// are made to an instance.
					if oldProviderAddr != "" {
						localAddr, diags := configs.ParseProviderConfigCompactStr(oldProviderAddr)
						if diags.HasErrors() {
							if strings.Contains(oldProviderAddr, "${") {
								// There seems to be a common misconception that
								// interpolation was valid in provider aliases
								// in 0.11, so we'll use a specialized error
								// message for that case.
								return nil, fmt.Errorf("invalid legacy provider config reference %q for %s: this alias seems to contain a template interpolation sequence, which was not supported but also not error-checked in Terraform 0.11. To proceed, rename the associated provider alias to a valid identifier and apply the change with Terraform 0.11 before upgrading to Terraform 0.12", oldProviderAddr, instAddr)
							}
							return nil, fmt.Errorf("invalid legacy provider config reference %q for %s: %s", oldProviderAddr, instAddr, diags.Err())
						}
						providerAddr = addrs.AbsProviderConfig{
							Module: moduleAddr.Module(),
							// We use NewLegacyProvider here so we can use
							// LegacyString() below to get the appropriate
							// legacy-style provider string.
							Provider: addrs.NewLegacyProvider(localAddr.LocalName),
							Alias:    localAddr.Alias,
						}
					} else {
						providerAddr = addrs.AbsProviderConfig{
							Module: moduleAddr.Module(),
							// We use NewLegacyProvider here so we can use
							// LegacyString() below to get the appropriate
							// legacy-style provider string.
							Provider: addrs.NewLegacyProvider(resAddr.ImpliedProvider()),
						}
					}
				}

				rs = &resourceStateV4{
					Module:         moduleAddr.String(),
					Mode:           modeStr,
					Type:           resAddr.Type,
					Name:           resAddr.Name,
					Instances:      []instanceObjectStateV4{},
					ProviderConfig: providerAddr.LegacyString(),
				}
				resourceStates[resAddr.String()] = rs
			}

			// Now we'll deal with the instance itself, which may either be
			// the first instance in a resource we just created or an additional
			// instance for a resource added on a prior loop.
			instKey := instAddr.Key
			if isOld := rsOld.Primary; isOld != nil {
				isNew, err := upgradeInstanceObjectV3ToV4(rsOld, isOld, instKey, states.NotDeposed)
				if err != nil {
					return nil, fmt.Errorf("failed to migrate primary generation of %s: %s", instAddr, err)
				}
				rs.Instances = append(rs.Instances, *isNew)
			}
			for i, isOld := range rsOld.Deposed {
				// When we migrate old instances we'll use sequential deposed
				// keys just so that the upgrade result is deterministic. New
				// deposed keys allocated moving forward will be pseudorandomly
				// selected, but we check for collisions and so these
				// non-random ones won't hurt.
				deposedKey := states.DeposedKey(fmt.Sprintf("%08x", i+1))
				isNew, err := upgradeInstanceObjectV3ToV4(rsOld, isOld, instKey, deposedKey)
				if err != nil {
					return nil, fmt.Errorf("failed to migrate deposed generation index %d of %s: %s", i, instAddr, err)
				}
				rs.Instances = append(rs.Instances, *isNew)
			}

			if instKey != addrs.NoKey && rs.EachMode == "" {
				rs.EachMode = "list"
			}
		}

		for _, rs := range resourceStates {
			new.Resources = append(new.Resources, *rs)
		}

		if len(msOld.Path) == 1 && msOld.Path[0] == "root" {
			// We'll migrate the outputs for this module too, then.
			for name, oldOS := range msOld.Outputs {
				newOS := outputStateV4{
					Sensitive: oldOS.Sensitive,
				}

				valRaw := oldOS.Value
				valSrc, err := json.Marshal(valRaw)
				if err != nil {
					// Should never happen, because this value came from JSON
					// in the first place and so we're just round-tripping here.
					return nil, fmt.Errorf("failed to serialize output %q value as JSON: %s", name, err)
				}

				// The "type" field in state V2 wasn't really that useful
				// since it was only able to capture string vs. list vs. map.
				// For this reason, during upgrade we'll just discard it
				// altogether and use cty's idea of the implied type of
				// turning our old value into JSON.
				ty, err := ctyjson.ImpliedType(valSrc)
				if err != nil {
					// REALLY should never happen, because we literally just
					// encoded this as JSON above!
					return nil, fmt.Errorf("failed to parse output %q value from JSON: %s", name, err)
				}

				// ImpliedType tends to produce structural types, but since older
				// version of Terraform didn't support those a collection type
				// is probably what was intended, so we'll see if we can
				// interpret our value as one.
				ty = simplifyImpliedValueType(ty)

				tySrc, err := ctyjson.MarshalType(ty)
				if err != nil {
					return nil, fmt.Errorf("failed to serialize output %q type as JSON: %s", name, err)
				}

				newOS.ValueRaw = json.RawMessage(valSrc)
				newOS.ValueTypeRaw = json.RawMessage(tySrc)

				new.RootOutputs[name] = newOS
			}
		}
	}

	new.normalize()

	return new, nil
}

func upgradeInstanceObjectV3ToV4(rsOld *resourceStateV2, isOld *instanceStateV2, instKey addrs.InstanceKey, deposedKey states.DeposedKey) (*instanceObjectStateV4, error) {

	// Schema versions were, in prior formats, a private concern of the provider
	// SDK, and not a first-class concept in the state format. Here we're
	// sniffing for the pre-0.12 SDK's way of representing schema versions
	// and promoting it to our first-class field if we find it. We'll ignore
	// it if it doesn't look like what the SDK would've written. If this
	// sniffing fails then we'll assume schema version 0.
	var schemaVersion uint64
	migratedSchemaVersion := false
	if raw, exists := isOld.Meta["schema_version"]; exists {
		switch tv := raw.(type) {
		case string:
			v, err := strconv.ParseUint(tv, 10, 64)
			if err == nil {
				schemaVersion = v
				migratedSchemaVersion = true
			}
		case int:
			schemaVersion = uint64(tv)
			migratedSchemaVersion = true
		case float64:
			schemaVersion = uint64(tv)
			migratedSchemaVersion = true
		}
	}

	private := map[string]interface{}{}
	for k, v := range isOld.Meta {
		if k == "schema_version" && migratedSchemaVersion {
			// We're gonna promote this into our first-class schema version field
			continue
		}
		private[k] = v
	}
	var privateJSON []byte
	if len(private) != 0 {
		var err error
		privateJSON, err = json.Marshal(private)
		if err != nil {
			// This shouldn't happen, because the Meta values all came from JSON
			// originally anyway.
			return nil, fmt.Errorf("cannot serialize private instance object data: %s", err)
		}
	}

	var status string
	if isOld.Tainted {
		status = "tainted"
	}

	var instKeyRaw interface{}
	switch tk := instKey.(type) {
	case addrs.IntKey:
		instKeyRaw = int(tk)
	case addrs.StringKey:
		instKeyRaw = string(tk)
	default:
		if instKeyRaw != nil {
			return nil, fmt.Errorf("unsupported instance key: %#v", instKey)
		}
	}

	var attributes map[string]string
	if isOld.Attributes != nil {
		attributes = make(map[string]string, len(isOld.Attributes))
		for k, v := range isOld.Attributes {
			attributes[k] = v
		}
	}
	if isOld.ID != "" {
		// As a special case, if we don't already have an "id" attribute and
		// yet there's a non-empty first-class ID on the old object then we'll
		// create a synthetic id attribute to avoid losing that first-class id.
		// In practice this generally arises only in tests where state literals
		// are hand-written in a non-standard way; real code prior to 0.12
		// would always force the first-class ID to be copied into the
		// id attribute before storing.
		if attributes == nil {
			attributes = make(map[string]string, len(isOld.Attributes))
		}
		if idVal := attributes["id"]; idVal == "" {
			attributes["id"] = isOld.ID
		}
	}

	dependencies := make([]string, 0, len(rsOld.Dependencies))
	for _, v := range rsOld.Dependencies {
		depStr, err := parseLegacyDependency(v)
		if err != nil {
			// We just drop invalid dependencies on the floor here, because
			// they tend to get left behind in Terraform 0.11 when resources
			// are renamed or moved between modules and there's no automatic
			// way to fix them here. In practice it shouldn't hurt to miss
			// a few dependency edges in the state because a subsequent plan
			// will run a refresh walk first and re-synchronize the
			// dependencies with the configuration.
			//
			// There is one rough edges where this can cause an incorrect
			// result, though: If the first command the user runs after
			// upgrading to Terraform 0.12 uses -refresh=false and thus
			// prevents the dependency reorganization from occurring _and_
			// that initial plan discovered "orphaned" resources (not present
			// in configuration any longer) then when the plan is applied the
			// destroy ordering will be incorrect for the instances of those
			// resources. We expect that is a rare enough situation that it
			// isn't a big deal, and even when it _does_ occur it's common for
			// the apply to succeed anyway unless many separate resources with
			// complex inter-dependencies are all orphaned at once.
			log.Printf("statefile: ignoring invalid dependency address %q while upgrading from state version 3 to version 4: %s", v, err)
			continue
		}
		dependencies = append(dependencies, depStr)
	}

	return &instanceObjectStateV4{
		IndexKey:       instKeyRaw,
		Status:         status,
		Deposed:        string(deposedKey),
		AttributesFlat: attributes,
		SchemaVersion:  schemaVersion,
		PrivateRaw:     privateJSON,
	}, nil
}

// parseLegacyResourceAddress parses the different identifier format used
// state formats before version 4, like "instance.name.0".
func parseLegacyResourceAddress(s string) (addrs.ResourceInstance, error) {
	var ret addrs.ResourceInstance

	// Split based on ".". Every resource address should have at least two
	// elements (type and name).
	parts := strings.Split(s, ".")
	if len(parts) < 2 || len(parts) > 4 {
		return ret, fmt.Errorf("invalid internal resource address format: %s", s)
	}

	// Data resource if we have at least 3 parts and the first one is data
	ret.Resource.Mode = addrs.ManagedResourceMode
	if len(parts) > 2 && parts[0] == "data" {
		ret.Resource.Mode = addrs.DataResourceMode
		parts = parts[1:]
	}

	// If we're not a data resource and we have more than 3, then it is an error
	if len(parts) > 3 && ret.Resource.Mode != addrs.DataResourceMode {
		return ret, fmt.Errorf("invalid internal resource address format: %s", s)
	}

	// Build the parts of the resource address that are guaranteed to exist
	ret.Resource.Type = parts[0]
	ret.Resource.Name = parts[1]
	ret.Key = addrs.NoKey

	// If we have more parts, then we have an index. Parse that.
	if len(parts) > 2 {
		idx, err := strconv.ParseInt(parts[2], 0, 0)
		if err != nil {
			return ret, fmt.Errorf("error parsing resource address %q: %s", s, err)
		}

		ret.Key = addrs.IntKey(idx)
	}

	return ret, nil
}

// simplifyImpliedValueType attempts to heuristically simplify a value type
// derived from a legacy stored output value into something simpler that
// is closer to what would've fitted into the pre-v0.12 value type system.
func simplifyImpliedValueType(ty cty.Type) cty.Type {
	switch {
	case ty.IsTupleType():
		// If all of the element types are the same then we'll make this
		// a list instead. This is very likely to be true, since prior versions
		// of Terraform did not officially support mixed-type collections.

		if ty.Equals(cty.EmptyTuple) {
			// Don't know what the element type would be, then.
			return ty
		}

		etys := ty.TupleElementTypes()
		ety := etys[0]
		for _, other := range etys[1:] {
			if !other.Equals(ety) {
				// inconsistent types
				return ty
			}
		}
		ety = simplifyImpliedValueType(ety)
		return cty.List(ety)

	case ty.IsObjectType():
		// If all of the attribute types are the same then we'll make this
		// a map instead. This is very likely to be true, since prior versions
		// of Terraform did not officially support mixed-type collections.

		if ty.Equals(cty.EmptyObject) {
			// Don't know what the element type would be, then.
			return ty
		}

		atys := ty.AttributeTypes()
		var ety cty.Type
		for _, other := range atys {
			if ety == cty.NilType {
				ety = other
				continue
			}
			if !other.Equals(ety) {
				// inconsistent types
				return ty
			}
		}
		ety = simplifyImpliedValueType(ety)
		return cty.Map(ety)

	default:
		// No other normalizations are possible
		return ty
	}
}

func parseLegacyDependency(s string) (string, error) {
	parts := strings.Split(s, ".")
	ret := parts[0]
	for _, part := range parts[1:] {
		if part == "*" {
			break
		}
		if i, err := strconv.Atoi(part); err == nil {
			ret = ret + fmt.Sprintf("[%d]", i)
			break
		}
		ret = ret + "." + part
	}

	// The result must parse as a reference, or else we'll create an invalid
	// state file.
	var diags tfdiags.Diagnostics
	_, diags = addrs.ParseRefStr(ret)
	if diags.HasErrors() {
		return "", diags.Err()
	}

	return ret, nil
}