core: Automatically upgrade resource instance states on read

If an instance object in state has an earlier schema version number then
it is likely that the schema we're holding won't be able to decode the
raw data that is stored. Instead, we must ask the provider to upgrade it
for us first, which might also include translating it from flatmap form
if it was last updated with a Terraform version earlier than v0.12.

This ends up being a "seam" between our use of int64 for schema versions
in the providers package and uint64 everywhere else. We intend to
standardize on int64 everywhere eventually, but for now this remains
consistent with existing usage in each layer to keep the type conversion
noise contained here and avoid mass-updates to other Terraform components
at this time.

This also includes a minor change to the test helpers for the
backend/local package, which were inexplicably setting a SchemaVersion of
1 on the basic test state but setting the mock schema version to zero,
creating an invalid situation where the state would need to be downgraded.

backend/local/backend_plan_test.go

@@ -320,8 +320,7 @@ func testPlanState() *states.State {
 			Name: "foo",
 		}.Instance(addrs.IntKey(0)),
 		&states.ResourceInstanceObjectSrc{
-			Status:        states.ObjectReady,
+			Status: states.ObjectReady,
-			SchemaVersion: 1,
 			AttrsJSON: []byte(`{
 				"ami": "bar",
 				"network_interface": [{

states/instance_object_src.go

@@ -89,3 +89,25 @@ func (os *ResourceInstanceObjectSrc) Decode(ty cty.Type) (*ResourceInstanceObjec
 		Private:      os.Private,
 	}, nil
 }
+
+// CompleteUpgrade creates a new ResourceInstanceObjectSrc by copying the
+// metadata from the receiver and writing in the given new schema version
+// and attribute value that are presumed to have resulted from upgrading
+// from an older schema version.
+func (os *ResourceInstanceObjectSrc) CompleteUpgrade(newAttrs cty.Value, newType cty.Type, newSchemaVersion uint64) (*ResourceInstanceObjectSrc, error) {
+	new := os.DeepCopy()
+	new.AttrsFlat = nil // We always use JSON after an upgrade, even if the source used flatmap
+
+	// This is the same principle as ResourceInstanceObject.Encode, but
+	// avoiding a decode/re-encode cycle because we don't have type info
+	// available for the "old" attributes.
+	newAttrs = cty.UnknownAsNull(newAttrs)
+	src, err := ctyjson.Marshal(newAttrs, newType)
+	if err != nil {
+		return nil, err
+	}
+
+	new.AttrsJSON = src
+	new.SchemaVersion = newSchemaVersion
+	return new, nil
+}

terraform/context_refresh_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/terraform/config/hcl2shim"
 	"github.com/hashicorp/terraform/configs/configschema"
 	"github.com/hashicorp/terraform/providers"
+	"github.com/hashicorp/terraform/states"
 )
 
 func TestContext2Refresh(t *testing.T) {
@@ -1567,3 +1568,169 @@ aws_instance.bar:
 		t.Fatalf("expected:\n%s\n\ngot:\n%s", expected, actual)
 	}
 }
+
+func TestContext2Refresh_schemaUpgradeFlatmap(t *testing.T) {
+	m := testModule(t, "empty")
+	p := testProvider("test")
+	p.GetSchemaReturn = &ProviderSchema{
+		ResourceTypes: map[string]*configschema.Block{
+			"test_thing": {
+				Attributes: map[string]*configschema.Attribute{
+					"name": { // imagining we renamed this from "id"
+						Type:     cty.String,
+						Optional: true,
+					},
+				},
+			},
+		},
+		ResourceTypeSchemaVersions: map[string]uint64{
+			"test_thing": 5,
+		},
+	}
+	p.UpgradeResourceStateResponse = providers.UpgradeResourceStateResponse{
+		UpgradedState: cty.ObjectVal(map[string]cty.Value{
+			"name": cty.StringVal("foo"),
+		}),
+	}
+
+	s := states.BuildState(func(s *states.SyncState) {
+		s.SetResourceInstanceCurrent(
+			addrs.Resource{
+				Mode: addrs.ManagedResourceMode,
+				Type: "test_thing",
+				Name: "bar",
+			}.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance),
+			&states.ResourceInstanceObjectSrc{
+				Status:        states.ObjectReady,
+				SchemaVersion: 3,
+				AttrsFlat: map[string]string{
+					"id": "foo",
+				},
+			},
+			addrs.ProviderConfig{Type: "test"}.Absolute(addrs.RootModuleInstance),
+		)
+	})
+
+	ctx := testContext2(t, &ContextOpts{
+		Config: m,
+		ProviderResolver: providers.ResolverFixed(
+			map[string]providers.Factory{
+				"test": testProviderFuncFixed(p),
+			},
+		),
+		State: s,
+	})
+
+	state, diags := ctx.Refresh()
+	if diags.HasErrors() {
+		t.Fatal(diags.Err())
+	}
+
+	{
+		got := p.UpgradeResourceStateRequest
+		want := providers.UpgradeResourceStateRequest{
+			TypeName: "test_thing",
+			Version:  3,
+			RawStateFlatmap: map[string]string{
+				"id": "foo",
+			},
+		}
+		if !cmp.Equal(got, want) {
+			t.Errorf("wrong upgrade request\n%s", cmp.Diff(want, got))
+		}
+	}
+
+	{
+		got := state.String()
+		want := strings.TrimSpace(`
+test_thing.bar:
+  ID = 
+  provider = provider.test
+  name = foo
+`)
+		if got != want {
+			t.Fatalf("wrong result state\ngot:\n%s\n\nwant:\n%s", got, want)
+		}
+	}
+}
+
+func TestContext2Refresh_schemaUpgradeJSON(t *testing.T) {
+	m := testModule(t, "empty")
+	p := testProvider("test")
+	p.GetSchemaReturn = &ProviderSchema{
+		ResourceTypes: map[string]*configschema.Block{
+			"test_thing": {
+				Attributes: map[string]*configschema.Attribute{
+					"name": { // imagining we renamed this from "id"
+						Type:     cty.String,
+						Optional: true,
+					},
+				},
+			},
+		},
+		ResourceTypeSchemaVersions: map[string]uint64{
+			"test_thing": 5,
+		},
+	}
+	p.UpgradeResourceStateResponse = providers.UpgradeResourceStateResponse{
+		UpgradedState: cty.ObjectVal(map[string]cty.Value{
+			"name": cty.StringVal("foo"),
+		}),
+	}
+
+	s := states.BuildState(func(s *states.SyncState) {
+		s.SetResourceInstanceCurrent(
+			addrs.Resource{
+				Mode: addrs.ManagedResourceMode,
+				Type: "test_thing",
+				Name: "bar",
+			}.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance),
+			&states.ResourceInstanceObjectSrc{
+				Status:        states.ObjectReady,
+				SchemaVersion: 3,
+				AttrsJSON:     []byte(`{"id":"foo"}`),
+			},
+			addrs.ProviderConfig{Type: "test"}.Absolute(addrs.RootModuleInstance),
+		)
+	})
+
+	ctx := testContext2(t, &ContextOpts{
+		Config: m,
+		ProviderResolver: providers.ResolverFixed(
+			map[string]providers.Factory{
+				"test": testProviderFuncFixed(p),
+			},
+		),
+		State: s,
+	})
+
+	state, diags := ctx.Refresh()
+	if diags.HasErrors() {
+		t.Fatal(diags.Err())
+	}
+
+	{
+		got := p.UpgradeResourceStateRequest
+		want := providers.UpgradeResourceStateRequest{
+			TypeName:     "test_thing",
+			Version:      3,
+			RawStateJSON: []byte(`{"id":"foo"}`),
+		}
+		if !cmp.Equal(got, want) {
+			t.Errorf("wrong upgrade request\n%s", cmp.Diff(want, got))
+		}
+	}
+
+	{
+		got := state.String()
+		want := strings.TrimSpace(`
+test_thing.bar:
+  ID = 
+  provider = provider.test
+  name = foo
+`)
+		if got != want {
+			t.Fatalf("wrong result state\ngot:\n%s\n\nwant:\n%s", got, want)
+		}
+	}
+}

terraform/eval_state.go

@@ -4,13 +4,11 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/hashicorp/terraform/tfdiags"
-
-	"github.com/hashicorp/terraform/configs"
-
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/providers"
 	"github.com/hashicorp/terraform/states"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
 // EvalReadState is an EvalNode implementation that reads the
@@ -50,9 +48,18 @@ func (n *EvalReadState) Eval(ctx EvalContext) (interface{}, error) {
 	}
 
 	schema, currentVersion := (*n.ProviderSchema).SchemaForResourceAddr(n.Addr.ContainingResource())
-	if src.SchemaVersion < currentVersion {
+	if schema == nil {
-		// TODO: Implement schema upgrades
+		// Shouldn't happen since we should've failed long ago if no schema is present
-		return nil, fmt.Errorf("schema upgrading is not yet implemented to take state from version %d to version %d", src.SchemaVersion, currentVersion)
+		return nil, fmt.Errorf("no schema available for %s while reading state; this is a bug in Terraform and should be reported", absAddr)
+	}
+	var diags tfdiags.Diagnostics
+	src, diags = UpgradeResourceState(absAddr, *n.Provider, src, schema, currentVersion)
+	if diags.HasErrors() {
+		// Note that we don't have any channel to return warnings here. We'll
+		// accept that for now since warnings during a schema upgrade would
+		// be pretty weird anyway, since this operation is supposed to seem
+		// invisible to the user.
+		return nil, diags.Err()
 	}
 
 	obj, err := src.Decode(schema.ImpliedType())
@@ -110,9 +117,18 @@ func (n *EvalReadStateDeposed) Eval(ctx EvalContext) (interface{}, error) {
 	}
 
 	schema, currentVersion := (*n.ProviderSchema).SchemaForResourceAddr(n.Addr.ContainingResource())
-	if src.SchemaVersion < currentVersion {
+	if schema == nil {
-		// TODO: Implement schema upgrades
+		// Shouldn't happen since we should've failed long ago if no schema is present
-		return nil, fmt.Errorf("schema upgrading is not yet implemented to take state from version %d to version %d", src.SchemaVersion, currentVersion)
+		return nil, fmt.Errorf("no schema available for %s while reading state; this is a bug in Terraform and should be reported", absAddr)
+	}
+	var diags tfdiags.Diagnostics
+	src, diags = UpgradeResourceState(absAddr, *n.Provider, src, schema, currentVersion)
+	if diags.HasErrors() {
+		// Note that we don't have any channel to return warnings here. We'll
+		// accept that for now since warnings during a schema upgrade would
+		// be pretty weird anyway, since this operation is supposed to seem
+		// invisible to the user.
+		return nil, diags.Err()
 	}
 
 	obj, err := src.Decode(schema.ImpliedType())

terraform/eval_state_upgrade.go

@@ -0,0 +1,104 @@
+package terraform
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/providers"
+	"github.com/hashicorp/terraform/states"
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+// UpgradeResourceState will, if necessary, run the provider-defined upgrade
+// logic against the given state object to make it compliant with the
+// current schema version. This is a no-op if the given state object is
+// already at the latest version.
+//
+// If any errors occur during upgrade, error diagnostics are returned. In that
+// case it is not safe to proceed with using the original state object.
+func UpgradeResourceState(addr addrs.AbsResourceInstance, provider providers.Interface, src *states.ResourceInstanceObjectSrc, currentSchema *configschema.Block, currentVersion uint64) (*states.ResourceInstanceObjectSrc, tfdiags.Diagnostics) {
+	if src.SchemaVersion == currentVersion {
+		// No upgrading required, then.
+		return src, nil
+	}
+	if addr.Resource.Resource.Mode != addrs.ManagedResourceMode {
+		// We only do state upgrading for managed resources.
+		return src, nil
+	}
+
+	providerType := addr.Resource.Resource.DefaultProviderConfig().Type
+	if src.SchemaVersion > currentVersion {
+		log.Printf("[TRACE] UpgradeResourceState: can't downgrade state for %s from version %d to %d", addr, src.SchemaVersion, currentVersion)
+		var diags tfdiags.Diagnostics
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Resource instance managed by newer provider version",
+			// This is not a very good error message, but we don't retain enough
+			// information in state to give good feedback on what provider
+			// version might be required here. :(
+			fmt.Sprintf("The current state of %s was created by a newer provider version than is currently selected. Upgrade the %s provider to work with this state.", addr, providerType),
+		))
+		return nil, diags
+	}
+
+	// If we get down here then we need to upgrade the state, with the
+	// provider's help.
+	// If this state was originally created by a version of Terraform prior to
+	// v0.12, this also includes translating from legacy flatmap to new-style
+	// representation, since only the provider has enough information to
+	// understand a flatmap built against an older schema.
+	log.Printf("[TRACE] UpgradeResourceState: upgrading state for %s from version %d to %d using provider %q", addr, src.SchemaVersion, currentVersion, providerType)
+
+	req := providers.UpgradeResourceStateRequest{
+		TypeName: addr.Resource.Resource.Type,
+
+		// TODO: The internal schema version representations are all using
+		// uint64 instead of int64, but unsigned integers aren't friendly
+		// to all protobuf target languages so in practice we use int64
+		// on the wire. In future we will change all of our internal
+		// representations to int64 too.
+		Version: int64(src.SchemaVersion),
+	}
+
+	if len(src.AttrsJSON) != 0 {
+		req.RawStateJSON = src.AttrsJSON
+	} else {
+		req.RawStateFlatmap = src.AttrsFlat
+	}
+
+	resp := provider.UpgradeResourceState(req)
+	diags := resp.Diagnostics
+	if diags.HasErrors() {
+		return nil, diags
+	}
+
+	// After upgrading, the new value must conform to the current schema. When
+	// going over RPC this is actually already ensured by the
+	// marshaling/unmarshaling of the new value, but we'll check it here
+	// anyway for robustness, e.g. for in-process providers.
+	newValue := resp.UpgradedState
+	if errs := newValue.Type().TestConformance(currentSchema.ImpliedType()); len(errs) > 0 {
+		for _, err := range errs {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Invalid resource state upgrade",
+				fmt.Sprintf("The %s provider upgraded the state for %s from a previous version, but produced an invalid result: %s.", providerType, addr, tfdiags.FormatError(err)),
+			))
+		}
+		return nil, diags
+	}
+
+	new, err := src.CompleteUpgrade(newValue, currentSchema.ImpliedType(), uint64(currentVersion))
+	if err != nil {
+		// We already checked for type conformance above, so getting into this
+		// codepath should be rare and is probably a bug somewhere under CompleteUpgrade.
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Failed to encode result of resource state upgrade",
+			fmt.Sprintf("Failed to encode state for %s after resource schema upgrade: %s.", addr, tfdiags.FormatError(err)),
+		))
+	}
+	return new, diags
+}

terraform/provider_mock.go

@@ -125,7 +125,8 @@ func (p *MockProvider) getSchema() providers.GetSchemaResponse {
 		}
 		for n, s := range p.GetSchemaReturn.ResourceTypes {
 			ret.ResourceTypes[n] = providers.Schema{
-				Block: s,
+				Version: int64(p.GetSchemaReturn.ResourceTypeSchemaVersions[n]),
+				Block:   s,
 			}
 		}
 	}

terraform/schemas.go

@@ -141,7 +141,7 @@ func loadProviderSchemas(schemas map[string]*ProviderSchema, config *configs.Con
 
 		for t, r := range resp.ResourceTypes {
 			s.ResourceTypes[t] = r.Block
-			s.ResourceTypeSchemaVersions[t] = r.Version
+			s.ResourceTypeSchemaVersions[t] = uint64(r.Version)
 			if r.Version < 0 {
 				diags = diags.Append(
 					fmt.Errorf("invalid negative schema version for resource type %s in provider %q", t, typeName),