Merge pull request #26947 from hashicorp/alisdair/backend-validate-remote-backend-terraform-version

backend: Validate remote backend Terraform version

backend/remote/backend.go

@@ -85,6 +85,12 @@ type Remote struct {
 
 	// opLock locks operations
 	opLock sync.Mutex
+
+	// ignoreVersionConflict, if true, will disable the requirement that the
+	// local Terraform version matches the remote workspace's configured
+	// version. This will also cause VerifyWorkspaceTerraformVersion to return
+	// a warning diagnostic instead of an error.
+	ignoreVersionConflict bool
 }
 
 var _ backend.Backend = (*Remote)(nil)
@@ -629,6 +635,17 @@ func (b *Remote) StateMgr(name string) (statemgr.Full, error) {
 		}
 	}
 
+	// This is a fallback error check. Most code paths should use other
+	// mechanisms to check the version, then set the ignoreVersionConflict
+	// field to true. This check is only in place to ensure that we don't
+	// accidentally upgrade state with a new code path, and the version check
+	// logic is coarser and simpler.
+	if !b.ignoreVersionConflict {
+		if workspace.TerraformVersion != tfversion.String() {
+			return nil, fmt.Errorf("Remote workspace Terraform version %q does not match local Terraform version %q", workspace.TerraformVersion, tfversion.String())
+		}
+	}
+
 	client := &remoteClient{
 		client:       b.client,
 		organization: b.organization,
@@ -676,9 +693,17 @@ func (b *Remote) Operation(ctx context.Context, op *backend.Operation) (*backend
 
 	// Check if we need to use the local backend to run the operation.
 	if b.forceLocal || !w.Operations {
+		if !w.Operations {
+			// Workspace is explicitly configured for local operations, so its
+			// configured Terraform version is meaningless
+			b.IgnoreVersionConflict()
+		}
 		return b.local.Operation(ctx, op)
 	}
 
+	// Running remotely so we don't care about version conflicts
+	b.IgnoreVersionConflict()
+
 	// Set the remote workspace name.
 	op.Workspace = w.Name
 
@@ -837,6 +862,101 @@ func (b *Remote) ReportResult(op *backend.RunningOperation, err error) {
 	}
 }
 
+// IgnoreVersionConflict allows commands to disable the fall-back check that
+// the local Terraform version matches the remote workspace's configured
+// Terraform version. This should be called by commands where this check is
+// unnecessary, such as those performing remote operations, or read-only
+// operations. It will also be called if the user uses a command-line flag to
+// override this check.
+func (b *Remote) IgnoreVersionConflict() {
+	b.ignoreVersionConflict = true
+}
+
+// VerifyWorkspaceTerraformVersion compares the local Terraform version against
+// the workspace's configured Terraform version. If they are equal, this means
+// that there are no compatibility concerns, so it returns no diagnostics.
+//
+// If the versions differ,
+func (b *Remote) VerifyWorkspaceTerraformVersion(workspaceName string) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+
+	workspace, err := b.getRemoteWorkspace(context.Background(), workspaceName)
+	if err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Error looking up workspace",
+			fmt.Sprintf("Workspace read failed: %s", err),
+		))
+		return diags
+	}
+
+	remoteVersion, err := version.NewSemver(workspace.TerraformVersion)
+	if err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Error looking up workspace",
+			fmt.Sprintf("Invalid Terraform version: %s", err),
+		))
+		return diags
+	}
+
+	v014 := version.Must(version.NewSemver("0.14.0"))
+	if tfversion.SemVer.LessThan(v014) || remoteVersion.LessThan(v014) {
+		// Versions of Terraform prior to 0.14.0 will refuse to load state files
+		// written by a newer version of Terraform, even if it is only a patch
+		// level difference. As a result we require an exact match.
+		if tfversion.SemVer.Equal(remoteVersion) {
+			return diags
+		}
+	}
+	if tfversion.SemVer.GreaterThanOrEqual(v014) && remoteVersion.GreaterThanOrEqual(v014) {
+		// Versions of Terraform after 0.14.0 should be compatible with each
+		// other.  At the time this code was written, the only constraints we
+		// are aware of are:
+		//
+		// - 0.14.0 is guaranteed to be compatible with versions up to but not
+		//   including 1.1.0
+		v110 := version.Must(version.NewSemver("1.1.0"))
+		if tfversion.SemVer.LessThan(v110) && remoteVersion.LessThan(v110) {
+			return diags
+		}
+		// - Any new Terraform state version will require at least minor patch
+		//   increment, so x.y.* will always be compatible with each other
+		tfvs := tfversion.SemVer.Segments64()
+		rwvs := remoteVersion.Segments64()
+		if len(tfvs) == 3 && len(rwvs) == 3 && tfvs[0] == rwvs[0] && tfvs[1] == rwvs[1] {
+			return diags
+		}
+	}
+
+	// Even if ignoring version conflicts, it may still be useful to call this
+	// method and warn the user about a mismatch between the local and remote
+	// Terraform versions.
+	severity := tfdiags.Error
+	if b.ignoreVersionConflict {
+		severity = tfdiags.Warning
+	}
+
+	suggestion := " If you're sure you want to upgrade the state, you can force Terraform to continue using the -ignore-remote-version flag. This may result in an unusable workspace."
+	if b.ignoreVersionConflict {
+		suggestion = ""
+	}
+	diags = diags.Append(tfdiags.Sourceless(
+		severity,
+		"Terraform version mismatch",
+		fmt.Sprintf(
+			"The local Terraform version (%s) does not match the configured version for remote workspace %s/%s (%s).%s",
+			tfversion.String(),
+			b.organization,
+			workspace.Name,
+			workspace.TerraformVersion,
+			suggestion,
+		),
+	))
+
+	return diags
+}
+
 // Colorize returns the Colorize structure that can be used for colorizing
 // output. This is guaranteed to always return a non-nil value and so useful
 // as a helper to wrap any potentially colored strings.

backend/remote/backend_apply_test.go

@@ -11,12 +11,14 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	tfe "github.com/hashicorp/go-tfe"
+	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/plans/planfile"
 	"github.com/hashicorp/terraform/states/statemgr"
 	"github.com/hashicorp/terraform/terraform"
+	tfversion "github.com/hashicorp/terraform/version"
 	"github.com/mitchellh/cli"
 )
 
@@ -1277,3 +1279,133 @@ func TestRemote_applyWithRemoteError(t *testing.T) {
 		t.Fatalf("expected apply error in output: %s", output)
 	}
 }
+
+func TestRemote_applyVersionCheck(t *testing.T) {
+	testCases := map[string]struct {
+		localVersion  string
+		remoteVersion string
+		forceLocal    bool
+		hasOperations bool
+		wantErr       string
+	}{
+		"versions can be different for remote apply": {
+			localVersion:  "0.14.0",
+			remoteVersion: "0.13.5",
+			hasOperations: true,
+		},
+		"versions can be different for local apply": {
+			localVersion:  "0.14.0",
+			remoteVersion: "0.13.5",
+			hasOperations: false,
+		},
+		"error if force local, has remote operations, different versions": {
+			localVersion:  "0.14.0",
+			remoteVersion: "0.13.5",
+			forceLocal:    true,
+			hasOperations: true,
+			wantErr:       `Remote workspace Terraform version "0.13.5" does not match local Terraform version "0.14.0"`,
+		},
+		"no error if versions are identical": {
+			localVersion:  "0.14.0",
+			remoteVersion: "0.14.0",
+			forceLocal:    true,
+			hasOperations: true,
+		},
+		"no error if force local but workspace has remote operations disabled": {
+			localVersion:  "0.14.0",
+			remoteVersion: "0.13.5",
+			forceLocal:    true,
+			hasOperations: false,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			b, bCleanup := testBackendDefault(t)
+			defer bCleanup()
+
+			// SETUP: Save original local version state and restore afterwards
+			p := tfversion.Prerelease
+			v := tfversion.Version
+			s := tfversion.SemVer
+			defer func() {
+				tfversion.Prerelease = p
+				tfversion.Version = v
+				tfversion.SemVer = s
+			}()
+
+			// SETUP: Set local version for the test case
+			tfversion.Prerelease = ""
+			tfversion.Version = tc.localVersion
+			tfversion.SemVer = version.Must(version.NewSemver(tc.localVersion))
+
+			// SETUP: Set force local for the test case
+			b.forceLocal = tc.forceLocal
+
+			ctx := context.Background()
+
+			// SETUP: set the operations and Terraform Version fields on the
+			// remote workspace
+			_, err := b.client.Workspaces.Update(
+				ctx,
+				b.organization,
+				b.workspace,
+				tfe.WorkspaceUpdateOptions{
+					Operations:       tfe.Bool(tc.hasOperations),
+					TerraformVersion: tfe.String(tc.remoteVersion),
+				},
+			)
+			if err != nil {
+				t.Fatalf("error creating named workspace: %v", err)
+			}
+
+			// RUN: prepare the apply operation and run it
+			op, configCleanup := testOperationApply(t, "./testdata/apply")
+			defer configCleanup()
+
+			input := testInput(t, map[string]string{
+				"approve": "yes",
+			})
+
+			op.UIIn = input
+			op.UIOut = b.CLI
+			op.Workspace = backend.DefaultStateName
+
+			run, err := b.Operation(ctx, op)
+			if err != nil {
+				t.Fatalf("error starting operation: %v", err)
+			}
+
+			// RUN: wait for completion
+			<-run.Done()
+
+			if tc.wantErr != "" {
+				// ASSERT: if the test case wants an error, check for failure
+				// and the error message
+				if run.Result != backend.OperationFailure {
+					t.Fatalf("expected run to fail, but result was %#v", run.Result)
+				}
+				errOutput := b.CLI.(*cli.MockUi).ErrorWriter.String()
+				if !strings.Contains(errOutput, tc.wantErr) {
+					t.Fatalf("missing error %q\noutput: %s", tc.wantErr, errOutput)
+				}
+			} else {
+				// ASSERT: otherwise, check for success and appropriate output
+				// based on whether the run should be local or remote
+				if run.Result != backend.OperationSuccess {
+					t.Fatalf("operation failed: %s", b.CLI.(*cli.MockUi).ErrorWriter.String())
+				}
+				output := b.CLI.(*cli.MockUi).OutputWriter.String()
+				hasRemote := strings.Contains(output, "Running apply in the remote backend")
+				if !tc.forceLocal && tc.hasOperations && !hasRemote {
+					t.Fatalf("missing remote backend header in output: %s", output)
+				} else if (tc.forceLocal || !tc.hasOperations) && hasRemote {
+					t.Fatalf("unexpected remote backend header in output: %s", output)
+				}
+				if !strings.Contains(output, "1 added, 0 changed, 0 destroyed") {
+					t.Fatalf("expected apply summary in output: %s", output)
+				}
+			}
+		})
+	}
+}

backend/remote/backend_context.go

@@ -156,11 +156,20 @@ func (b *Remote) getRemoteWorkspaceName(localWorkspaceName string) string {
 	}
 }
 
-func (b *Remote) getRemoteWorkspaceID(ctx context.Context, localWorkspaceName string) (string, error) {
+func (b *Remote) getRemoteWorkspace(ctx context.Context, localWorkspaceName string) (*tfe.Workspace, error) {
 	remoteWorkspaceName := b.getRemoteWorkspaceName(localWorkspaceName)
 
-	log.Printf("[TRACE] backend/remote: looking up workspace id for %s/%s", b.organization, remoteWorkspaceName)
+	log.Printf("[TRACE] backend/remote: looking up workspace for %s/%s", b.organization, remoteWorkspaceName)
 	remoteWorkspace, err := b.client.Workspaces.Read(ctx, b.organization, remoteWorkspaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	return remoteWorkspace, nil
+}
+
+func (b *Remote) getRemoteWorkspaceID(ctx context.Context, localWorkspaceName string) (string, error) {
+	remoteWorkspace, err := b.getRemoteWorkspace(ctx, localWorkspaceName)
 	if err != nil {
 		return "", err
 	}

backend/remote/backend_mock.go

@@ -17,6 +17,7 @@ import (
 
 	tfe "github.com/hashicorp/go-tfe"
 	"github.com/hashicorp/terraform/terraform"
+	tfversion "github.com/hashicorp/terraform/version"
 	"github.com/mitchellh/copystructure"
 )
 
@@ -1124,10 +1125,15 @@ func (m *mockWorkspaces) List(ctx context.Context, organization string, options
 }
 
 func (m *mockWorkspaces) Create(ctx context.Context, organization string, options tfe.WorkspaceCreateOptions) (*tfe.Workspace, error) {
+	if strings.HasSuffix(*options.Name, "no-operations") {
+		options.Operations = tfe.Bool(false)
+	} else if options.Operations == nil {
+		options.Operations = tfe.Bool(true)
+	}
 	w := &tfe.Workspace{
 		ID:         generateID("ws-"),
 		Name:       *options.Name,
-		Operations: !strings.HasSuffix(*options.Name, "no-operations"),
+		Operations: *options.Operations,
 		Permissions: &tfe.WorkspacePermissions{
 			CanQueueApply: true,
 			CanQueueRun:   true,
@@ -1139,6 +1145,11 @@ func (m *mockWorkspaces) Create(ctx context.Context, organization string, option
 	if options.VCSRepo != nil {
 		w.VCSRepo = &tfe.VCSRepo{}
 	}
+	if options.TerraformVersion != nil {
+		w.TerraformVersion = *options.TerraformVersion
+	} else {
+		w.TerraformVersion = tfversion.String()
+	}
 	m.workspaceIDs[w.ID] = w
 	m.workspaceNames[w.Name] = w
 	return w, nil
@@ -1171,6 +1182,9 @@ func (m *mockWorkspaces) Update(ctx context.Context, organization, workspace str
 		return nil, tfe.ErrResourceNotFound
 	}
 
+	if options.Operations != nil {
+		w.Operations = *options.Operations
+	}
 	if options.Name != nil {
 		w.Name = *options.Name
 	}

backend/remote/backend_test.go

@@ -1,13 +1,18 @@
 package remote
 
 import (
+	"context"
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
 
+	tfe "github.com/hashicorp/go-tfe"
+	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform-svchost/disco"
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/version"
+	"github.com/hashicorp/terraform/tfdiags"
+	tfversion "github.com/hashicorp/terraform/version"
 	"github.com/zclconf/go-cty/cty"
 
 	backendLocal "github.com/hashicorp/terraform/backend/local"
@@ -196,11 +201,11 @@ func TestRemote_versionConstraints(t *testing.T) {
 	}
 
 	// Save and restore the actual version.
-	p := version.Prerelease
+	p := tfversion.Prerelease
-	v := version.Version
+	v := tfversion.Version
 	defer func() {
-		version.Prerelease = p
+		tfversion.Prerelease = p
-		version.Version = v
+		tfversion.Version = v
 	}()
 
 	for name, tc := range cases {
@@ -208,8 +213,8 @@ func TestRemote_versionConstraints(t *testing.T) {
 		b := New(testDisco(s))
 
 		// Set the version for this test.
-		version.Prerelease = tc.prerelease
+		tfversion.Prerelease = tc.prerelease
-		version.Version = tc.version
+		tfversion.Version = tc.version
 
 		// Validate
 		_, valDiags := b.PrepareConfig(tc.config)
@@ -428,17 +433,17 @@ func TestRemote_checkConstraints(t *testing.T) {
 	}
 
 	// Save and restore the actual version.
-	p := version.Prerelease
+	p := tfversion.Prerelease
-	v := version.Version
+	v := tfversion.Version
 	defer func() {
-		version.Prerelease = p
+		tfversion.Prerelease = p
-		version.Version = v
+		tfversion.Version = v
 	}()
 
 	for name, tc := range cases {
 		// Set the version for this test.
-		version.Prerelease = tc.prerelease
+		tfversion.Prerelease = tc.prerelease
-		version.Version = tc.version
+		tfversion.Version = tc.version
 
 		// Check the constraints.
 		diags := b.checkConstraints(tc.constraints)
@@ -448,3 +453,222 @@ func TestRemote_checkConstraints(t *testing.T) {
 		}
 	}
 }
+
+func TestRemote_StateMgr_versionCheck(t *testing.T) {
+	b, bCleanup := testBackendDefault(t)
+	defer bCleanup()
+
+	// Some fixed versions for testing with. This logic is a simple string
+	// comparison, so we don't need many test cases.
+	v0135 := version.Must(version.NewSemver("0.13.5"))
+	v0140 := version.Must(version.NewSemver("0.14.0"))
+
+	// Save original local version state and restore afterwards
+	p := tfversion.Prerelease
+	v := tfversion.Version
+	s := tfversion.SemVer
+	defer func() {
+		tfversion.Prerelease = p
+		tfversion.Version = v
+		tfversion.SemVer = s
+	}()
+
+	// For this test, the local Terraform version is set to 0.14.0
+	tfversion.Prerelease = ""
+	tfversion.Version = v0140.String()
+	tfversion.SemVer = v0140
+
+	// Update the mock remote workspace Terraform version to match the local
+	// Terraform version
+	if _, err := b.client.Workspaces.Update(
+		context.Background(),
+		b.organization,
+		b.workspace,
+		tfe.WorkspaceUpdateOptions{
+			TerraformVersion: tfe.String(v0140.String()),
+		},
+	); err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	// This should succeed
+	if _, err := b.StateMgr(backend.DefaultStateName); err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	// Now change the remote workspace to a different Terraform version
+	if _, err := b.client.Workspaces.Update(
+		context.Background(),
+		b.organization,
+		b.workspace,
+		tfe.WorkspaceUpdateOptions{
+			TerraformVersion: tfe.String(v0135.String()),
+		},
+	); err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	// This should fail
+	want := `Remote workspace Terraform version "0.13.5" does not match local Terraform version "0.14.0"`
+	if _, err := b.StateMgr(backend.DefaultStateName); err.Error() != want {
+		t.Fatalf("wrong error\n got: %v\nwant: %v", err.Error(), want)
+	}
+}
+
+func TestRemote_VerifyWorkspaceTerraformVersion(t *testing.T) {
+	testCases := []struct {
+		local   string
+		remote  string
+		wantErr bool
+	}{
+		{"0.13.5", "0.13.5", false},
+		{"0.14.0", "0.13.5", true},
+		{"0.14.0", "0.14.1", false},
+		{"0.14.0", "1.0.99", false},
+		{"0.14.0", "1.1.0", true},
+		{"1.2.0", "1.2.99", false},
+		{"1.2.0", "1.3.0", true},
+	}
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("local %s, remote %s", tc.local, tc.remote), func(t *testing.T) {
+			b, bCleanup := testBackendDefault(t)
+			defer bCleanup()
+
+			local := version.Must(version.NewSemver(tc.local))
+			remote := version.Must(version.NewSemver(tc.remote))
+
+			// Save original local version state and restore afterwards
+			p := tfversion.Prerelease
+			v := tfversion.Version
+			s := tfversion.SemVer
+			defer func() {
+				tfversion.Prerelease = p
+				tfversion.Version = v
+				tfversion.SemVer = s
+			}()
+
+			// Override local version as specified
+			tfversion.Prerelease = ""
+			tfversion.Version = local.String()
+			tfversion.SemVer = local
+
+			// Update the mock remote workspace Terraform version to the
+			// specified remote version
+			if _, err := b.client.Workspaces.Update(
+				context.Background(),
+				b.organization,
+				b.workspace,
+				tfe.WorkspaceUpdateOptions{
+					TerraformVersion: tfe.String(remote.String()),
+				},
+			); err != nil {
+				t.Fatalf("error: %v", err)
+			}
+
+			diags := b.VerifyWorkspaceTerraformVersion(backend.DefaultStateName)
+			if tc.wantErr {
+				if len(diags) != 1 {
+					t.Fatal("expected diag, but none returned")
+				}
+				if got := diags.Err().Error(); !strings.Contains(got, "Terraform version mismatch") {
+					t.Fatalf("unexpected error: %s", got)
+				}
+			} else {
+				if len(diags) != 0 {
+					t.Fatalf("unexpected diags: %s", diags.Err())
+				}
+			}
+		})
+	}
+}
+
+func TestRemote_VerifyWorkspaceTerraformVersion_workspaceErrors(t *testing.T) {
+	b, bCleanup := testBackendDefault(t)
+	defer bCleanup()
+
+	// Attempting to check the version against a workspace which doesn't exist
+	// should fail
+	diags := b.VerifyWorkspaceTerraformVersion("invalid-workspace")
+	if len(diags) != 1 {
+		t.Fatal("expected diag, but none returned")
+	}
+	if got := diags.Err().Error(); !strings.Contains(got, "Error looking up workspace: Workspace read failed") {
+		t.Fatalf("unexpected error: %s", got)
+	}
+
+	// Update the mock remote workspace Terraform version to an invalid version
+	if _, err := b.client.Workspaces.Update(
+		context.Background(),
+		b.organization,
+		b.workspace,
+		tfe.WorkspaceUpdateOptions{
+			TerraformVersion: tfe.String("1.0.cheetarah"),
+		},
+	); err != nil {
+		t.Fatalf("error: %v", err)
+	}
+	diags = b.VerifyWorkspaceTerraformVersion(backend.DefaultStateName)
+
+	if len(diags) != 1 {
+		t.Fatal("expected diag, but none returned")
+	}
+	if got := diags.Err().Error(); !strings.Contains(got, "Error looking up workspace: Invalid Terraform version") {
+		t.Fatalf("unexpected error: %s", got)
+	}
+}
+
+func TestRemote_VerifyWorkspaceTerraformVersion_ignoreFlagSet(t *testing.T) {
+	b, bCleanup := testBackendDefault(t)
+	defer bCleanup()
+
+	// If the ignore flag is set, the behaviour changes
+	b.IgnoreVersionConflict()
+
+	// Different local & remote versions to cause an error
+	local := version.Must(version.NewSemver("0.14.0"))
+	remote := version.Must(version.NewSemver("0.13.5"))
+
+	// Save original local version state and restore afterwards
+	p := tfversion.Prerelease
+	v := tfversion.Version
+	s := tfversion.SemVer
+	defer func() {
+		tfversion.Prerelease = p
+		tfversion.Version = v
+		tfversion.SemVer = s
+	}()
+
+	// Override local version as specified
+	tfversion.Prerelease = ""
+	tfversion.Version = local.String()
+	tfversion.SemVer = local
+
+	// Update the mock remote workspace Terraform version to the
+	// specified remote version
+	if _, err := b.client.Workspaces.Update(
+		context.Background(),
+		b.organization,
+		b.workspace,
+		tfe.WorkspaceUpdateOptions{
+			TerraformVersion: tfe.String(remote.String()),
+		},
+	); err != nil {
+		t.Fatalf("error: %v", err)
+	}
+
+	diags := b.VerifyWorkspaceTerraformVersion(backend.DefaultStateName)
+	if len(diags) != 1 {
+		t.Fatal("expected diag, but none returned")
+	}
+
+	if got, want := diags[0].Severity(), tfdiags.Warning; got != want {
+		t.Errorf("wrong severity: got %#v, want %#v", got, want)
+	}
+	if got, want := diags[0].Description().Summary, "Terraform version mismatch"; got != want {
+		t.Errorf("wrong summary: got %s, want %s", got, want)
+	}
+	wantDetail := "The local Terraform version (0.14.0) does not match the configured version for remote workspace hashicorp/prod (0.13.5)."
+	if got := diags[0].Description().Detail; got != wantDetail {
+		t.Errorf("wrong summary: got %s, want %s", got, wantDetail)
+	}
+}

command/console.go

@@ -69,6 +69,9 @@ func (c *ConsoleCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Build the operation
 	opReq := c.Operation(b)
 	opReq.ConfigDir = configPath

command/graph.go

@@ -87,6 +87,9 @@ func (c *GraphCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Build the operation
 	opReq := c.Operation(b)
 	opReq.ConfigDir = configPath

command/import.go

@@ -35,6 +35,7 @@ func (c *ImportCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 
 	cmdFlags := c.Meta.extendedFlagSet("import")
+	cmdFlags.BoolVar(&c.ignoreRemoteVersion, "ignore-remote-version", false, "continue even if remote and local Terraform versions differ")
 	cmdFlags.IntVar(&c.Meta.parallelism, "parallelism", DefaultParallelism, "parallelism")
 	cmdFlags.StringVar(&c.Meta.statePath, "state", "", "path")
 	cmdFlags.StringVar(&c.Meta.stateOutPath, "state-out", "", "path")
@@ -198,6 +199,14 @@ func (c *ImportCommand) Run(args []string) int {
 		}
 	}
 
+	// Check remote Terraform version is compatible
+	remoteVersionDiags := c.remoteBackendVersionCheck(b, opReq.Workspace)
+	diags = diags.Append(remoteVersionDiags)
+	c.showDiagnostics(diags)
+	if diags.HasErrors() {
+		return 1
+	}
+
 	// Get the context
 	ctx, state, ctxDiags := local.Context(opReq)
 	diags = diags.Append(ctxDiags)
@@ -321,6 +330,9 @@ Options:
                           a file. If "terraform.tfvars" or any ".auto.tfvars"
                           files are present, they will be automatically loaded.
 
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
 
 `
 	return strings.TrimSpace(helpText)

command/init.go

@@ -264,6 +264,7 @@ func (c *InitCommand) Run(args []string) int {
 	// on a previous run) we'll use the current state as a potential source
 	// of provider dependencies.
 	if back != nil {
+		c.ignoreRemoteBackendVersionConflict(back)
 		workspace, err := c.Workspace()
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))

command/meta.go

@@ -205,6 +205,10 @@ type Meta struct {
 
 	// Used with the import command to allow import of state when no matching config exists.
 	allowMissingConfig bool
+
+	// Used with commands which write state to allow users to write remote
+	// state even if the remote and local Terraform versions don't match.
+	ignoreRemoteVersion bool
 }
 
 type testingOverrides struct {
@@ -466,6 +470,17 @@ func (m *Meta) defaultFlagSet(n string) *flag.FlagSet {
 	return f
 }
 
+// ignoreRemoteVersionFlagSet add the ignore-remote version flag to suppress
+// the error when the configured Terraform version on the remote workspace
+// does not match the local Terraform version.
+func (m *Meta) ignoreRemoteVersionFlagSet(n string) *flag.FlagSet {
+	f := m.defaultFlagSet(n)
+
+	f.BoolVar(&m.ignoreRemoteVersion, "ignore-remote-version", false, "continue even if remote and local Terraform versions differ")
+
+	return f
+}
+
 // extendedFlagSet adds custom flags that are mostly used by commands
 // that are used to run an operation like plan or apply.
 func (m *Meta) extendedFlagSet(n string) *flag.FlagSet {

command/meta_backend.go

@@ -17,6 +17,7 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/terraform/backend"
+	remoteBackend "github.com/hashicorp/terraform/backend/remote"
 	"github.com/hashicorp/terraform/command/clistate"
 	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/plans"
@@ -1091,6 +1092,33 @@ func (m *Meta) backendInitRequired(reason string) {
 		"[reset]"+strings.TrimSpace(errBackendInit)+"\n", reason)))
 }
 
+// Helper method to ignore remote backend version conflicts. Only call this
+// for commands which cannot accidentally upgrade remote state files.
+func (m *Meta) ignoreRemoteBackendVersionConflict(b backend.Backend) {
+	if rb, ok := b.(*remoteBackend.Remote); ok {
+		rb.IgnoreVersionConflict()
+	}
+}
+
+// Helper method to check the local Terraform version against the configured
+// version in the remote workspace, returning diagnostics if they conflict.
+func (m *Meta) remoteBackendVersionCheck(b backend.Backend, workspace string) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
+
+	if rb, ok := b.(*remoteBackend.Remote); ok {
+		// Allow user override based on command-line flag
+		if m.ignoreRemoteVersion {
+			rb.IgnoreVersionConflict()
+		}
+		// If the override is set, this check will return a warning instead of
+		// an error
+		versionDiags := rb.VerifyWorkspaceTerraformVersion(workspace)
+		diags = diags.Append(versionDiags)
+	}
+
+	return diags
+}
+
 //-------------------------------------------------------------------
 // Output constants and initialization code
 //-------------------------------------------------------------------

command/output.go

@@ -61,6 +61,9 @@ func (c *OutputCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	env, err := c.Workspace()
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))

command/providers.go

@@ -82,6 +82,9 @@ func (c *ProvidersCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Get the state
 	env, err := c.Workspace()
 	if err != nil {

command/providers_schema.go

@@ -67,6 +67,9 @@ func (c *ProvidersSchemaCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// we expect that the config dir is the cwd
 	cwd, err := os.Getwd()
 	if err != nil {

command/show.go

@@ -68,6 +68,9 @@ func (c *ShowCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// the show command expects the config dir to always be the cwd
 	cwd, err := os.Getwd()
 	if err != nil {

command/state_list.go

@@ -40,6 +40,9 @@ func (c *StateListCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Get the state
 	env, err := c.Workspace()
 	if err != nil {

command/state_meta.go

@@ -41,6 +41,14 @@ func (c *StateMeta) State() (statemgr.Full, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		// Check remote Terraform version is compatible
+		remoteVersionDiags := c.remoteBackendVersionCheck(b, workspace)
+		c.showDiagnostics(remoteVersionDiags)
+		if remoteVersionDiags.HasErrors() {
+			return nil, fmt.Errorf("Error checking remote Terraform version")
+		}
+
 		// Get the state
 		s, err := b.StateMgr(workspace)
 		if err != nil {

command/state_mv.go

@@ -23,7 +23,7 @@ func (c *StateMvCommand) Run(args []string) int {
 	var backupPathOut, statePathOut string
 
 	var dryRun bool
-	cmdFlags := c.Meta.defaultFlagSet("state mv")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("state mv")
 	cmdFlags.BoolVar(&dryRun, "dry-run", false, "dry run")
 	cmdFlags.StringVar(&c.backupPath, "backup", "-", "backup")
 	cmdFlags.StringVar(&backupPathOut, "backup-out", "-", "backup")
@@ -468,28 +468,32 @@ Options:
   -dry-run                If set, prints out what would've been moved but doesn't
                           actually move anything.
 
-  -backup=PATH        Path where Terraform should write the backup for the original
+  -backup=PATH            Path where Terraform should write the backup for the
-                      state. This can't be disabled. If not set, Terraform
+						  original state. This can't be disabled. If not set,
-                      will write it to the same path as the statefile with
+						  Terraform will write it to the same path as the
-                      a ".backup" extension.
+						  statefile with a ".backup" extension.
 
-  -backup-out=PATH    Path where Terraform should write the backup for the destination
+  -backup-out=PATH        Path where Terraform should write the backup for the
-                      state. This can't be disabled. If not set, Terraform
+						  destination state. This can't be disabled. If not
-                      will write it to the same path as the destination state
+						  set, Terraform will write it to the same path as the
-                      file with a backup extension. This only needs
+						  destination state file with a backup extension. This
-                      to be specified if -state-out is set to a different path
+						  only needs to be specified if -state-out is set to a
-                      than -state.
+						  different path than -state.
 
   -lock=true              Lock the state files when locking is supported.
 
   -lock-timeout=0s        Duration to retry a state lock.
 
-  -state=PATH         Path to the source state file. Defaults to the configured
+  -state=PATH             Path to the source state file. Defaults to the
-                      backend, or "terraform.tfstate"
+						  configured backend, or "terraform.tfstate"
 
-  -state-out=PATH     Path to the destination state file to write to. If this
+  -state-out=PATH         Path to the destination state file to write to. If
-                      isn't specified, the source state file will be used. This
+						  this isn't specified, the source state file will be
-                      can be a new or existing path.
+						  used. This can be a new or existing path.
+
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
 
 `
 	return strings.TrimSpace(helpText)

command/state_pull.go

@@ -31,6 +31,9 @@ func (c *StatePullCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Get the state manager for the current workspace
 	env, err := c.Workspace()
 	if err != nil {

command/state_push.go

@@ -22,7 +22,7 @@ type StatePushCommand struct {
 func (c *StatePushCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 	var flagForce bool
-	cmdFlags := c.Meta.defaultFlagSet("state push")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("state push")
 	cmdFlags.BoolVar(&flagForce, "force", false, "")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
 	cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout")
@@ -71,13 +71,22 @@ func (c *StatePushCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Get the state manager for the currently-selected workspace
+	// Determine the workspace name
-	env, err := c.Workspace()
+	workspace, err := c.Workspace()
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))
 		return 1
 	}
-	stateMgr, err := b.StateMgr(env)
+
+	// Check remote Terraform version is compatible
+	remoteVersionDiags := c.remoteBackendVersionCheck(b, workspace)
+	c.showDiagnostics(remoteVersionDiags)
+	if remoteVersionDiags.HasErrors() {
+		return 1
+	}
+
+	// Get the state manager for the currently-selected workspace
+	stateMgr, err := b.StateMgr(workspace)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to load destination state: %s", err))
 		return 1

command/state_replace_provider.go

@@ -25,7 +25,7 @@ func (c *StateReplaceProviderCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 
 	var autoApprove bool
-	cmdFlags := c.Meta.defaultFlagSet("state replace-provider")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("state replace-provider")
 	cmdFlags.BoolVar(&autoApprove, "auto-approve", false, "skip interactive approval of replacements")
 	cmdFlags.StringVar(&c.backupPath, "backup", "-", "backup")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock states")
@@ -175,16 +175,21 @@ Options:
   -auto-approve           Skip interactive approval.
 
   -backup=PATH            Path where Terraform should write the backup for the
-                      state file. This can't be disabled. If not set, Terraform
+						  state file. This can't be disabled. If not set,
-                      will write it to the same path as the state file with
+						  Terraform will write it to the same path as the state
-                      a ".backup" extension.
+						  file with a ".backup" extension.
 
   -lock=true              Lock the state files when locking is supported.
 
   -lock-timeout=0s        Duration to retry a state lock.
 
-  -state=PATH         Path to the state file to update. Defaults to the configured
+  -state=PATH             Path to the state file to update. Defaults to the
-                      backend, or "terraform.tfstate"
+						  configured backend, or "terraform.tfstate"
+
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
+
 `
 	return strings.TrimSpace(helpText)
 }

command/state_rm.go

@@ -19,7 +19,7 @@ type StateRmCommand struct {
 func (c *StateRmCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 	var dryRun bool
-	cmdFlags := c.Meta.defaultFlagSet("state rm")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("state rm")
 	cmdFlags.BoolVar(&dryRun, "dry-run", false, "dry run")
 	cmdFlags.StringVar(&c.backupPath, "backup", "-", "backup")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
@@ -156,8 +156,12 @@ Options:
 
   -lock-timeout=0s        Duration to retry a state lock.
 
-  -state=PATH         Path to the state file to update. Defaults to the current
+  -state=PATH             Path to the state file to update. Defaults to the
-                      workspace state.
+                          current workspace state.
+
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
 
 `
 	return strings.TrimSpace(helpText)

command/state_show.go

@@ -53,6 +53,9 @@ func (c *StateShowCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This is a read-only command
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	// Check if the address can be parsed
 	addr, addrDiags := addrs.ParseAbsResourceInstanceStr(args[0])
 	if addrDiags.HasErrors() {

command/taint.go

@@ -23,7 +23,7 @@ func (c *TaintCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 	var module string
 	var allowMissing bool
-	cmdFlags := c.Meta.defaultFlagSet("taint")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("taint")
 	cmdFlags.BoolVar(&allowMissing, "allow-missing", false, "module")
 	cmdFlags.StringVar(&c.Meta.backupPath, "backup", "", "path")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
@@ -100,13 +100,23 @@ func (c *TaintCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Get the state
+	// Determine the workspace name
-	env, err := c.Workspace()
+	workspace, err := c.Workspace()
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))
 		return 1
 	}
-	stateMgr, err := b.StateMgr(env)
+
+	// Check remote Terraform version is compatible
+	remoteVersionDiags := c.remoteBackendVersionCheck(b, workspace)
+	diags = diags.Append(remoteVersionDiags)
+	c.showDiagnostics(diags)
+	if diags.HasErrors() {
+		return 1
+	}
+
+	// Get the state
+	stateMgr, err := b.StateMgr(workspace)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to load state: %s", err))
 		return 1
@@ -241,6 +251,10 @@ Options:
   -state-out=path         Path to write updated state file. By default, the
                           "-state" path will be used.
 
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
+
 `
 	return strings.TrimSpace(helpText)
 }

command/untaint.go

@@ -21,7 +21,7 @@ func (c *UntaintCommand) Run(args []string) int {
 	args = c.Meta.process(args)
 	var module string
 	var allowMissing bool
-	cmdFlags := c.Meta.defaultFlagSet("untaint")
+	cmdFlags := c.Meta.ignoreRemoteVersionFlagSet("untaint")
 	cmdFlags.BoolVar(&allowMissing, "allow-missing", false, "module")
 	cmdFlags.StringVar(&c.Meta.backupPath, "backup", "", "path")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
@@ -65,12 +65,22 @@ func (c *UntaintCommand) Run(args []string) int {
 		return 1
 	}
 
-	// Get the state
+	// Determine the workspace name
 	workspace, err := c.Workspace()
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))
 		return 1
 	}
+
+	// Check remote Terraform version is compatible
+	remoteVersionDiags := c.remoteBackendVersionCheck(b, workspace)
+	diags = diags.Append(remoteVersionDiags)
+	c.showDiagnostics(diags)
+	if diags.HasErrors() {
+		return 1
+	}
+
+	// Get the state
 	stateMgr, err := b.StateMgr(workspace)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to load state: %s", err))
@@ -200,9 +210,9 @@ Options:
 
   -lock-timeout=0s        Duration to retry a state lock.
 
-  -module=path        The module path where the resource lives. By
+  -module=path            The module path where the resource lives. By default
-                      default this will be root. Child modules can be specified
+						  this will be root. Child modules can be specified by
-                      by names. Ex. "consul" or "consul.vpc" (nested modules).
+						  names. Ex. "consul" or "consul.vpc" (nested modules).
 
   -state=path             Path to read and save state (unless state-out
                           is specified). Defaults to "terraform.tfstate".
@@ -210,6 +220,10 @@ Options:
   -state-out=path         Path to write updated state file. By default, the
                           "-state" path will be used.
 
+  -ignore-remote-version  Continue even if remote and local Terraform versions
+                          differ. This may result in an unusable workspace, and
+                          should be used with extreme caution.
+
 `
 	return strings.TrimSpace(helpText)
 }

command/workspace_delete.go

@@ -65,6 +65,9 @@ func (c *WorkspaceDeleteCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This command will not write state
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	workspaces, err := b.Workspaces()
 	if err != nil {
 		c.Ui.Error(err.Error())

command/workspace_list.go

@@ -51,6 +51,9 @@ func (c *WorkspaceListCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This command will not write state
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	states, err := b.Workspaces()
 	if err != nil {
 		c.Ui.Error(err.Error())

command/workspace_new.go

@@ -81,6 +81,9 @@ func (c *WorkspaceNewCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This command will not write state
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	workspaces, err := b.Workspaces()
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to get configured named states: %s", err))

command/workspace_select.go

@@ -67,6 +67,9 @@ func (c *WorkspaceSelectCommand) Run(args []string) int {
 		return 1
 	}
 
+	// This command will not write state
+	c.ignoreRemoteBackendVersionConflict(b)
+
 	name := args[0]
 	if !validWorkspaceName(name) {
 		c.Ui.Error(fmt.Sprintf(envInvalidName, name))

website/docs/commands/import.html.md

@@ -87,6 +87,11 @@ in the configuration for the target resource, and that is the best behavior in m
   the working directory. This flag can be used multiple times. This is only
   useful with the `-config` flag.
 
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.
+
 ## Provider Configuration
 
 Terraform will attempt to load configuration files that configure the

website/docs/commands/state/mv.html.md

@@ -57,6 +57,11 @@ The command-line flags are all optional. The list of available flags are:
   isn't specified the source state file will be used. This can be a new or
   existing path.
 
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.
+
 ## Example: Rename a Resource
 
 The example below renames the `packet_device` resource named `worker` to `helper`:

website/docs/commands/state/push.html.md

@@ -42,3 +42,10 @@ making changes that appear to be unsafe:
 Both of these safety checks can be disabled with the `-force` flag.
 **This is not recommended.** If you disable the safety checks and are
 pushing state, the destination state will be overwritten.
+
+Other available flags:
+
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.

website/docs/commands/state/replace-provider.html.md

@@ -38,6 +38,11 @@ The command-line flags are all optional. The list of available flags are:
 * `-state=path` - Path to the source state file to read from. Defaults to the
   configured backend, or "terraform.tfstate".
 
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.
+
 ## Example
 
 The example below replaces the `hashicorp/aws` provider with a fork by `acme`, hosted at a private registry at `registry.acme.corp`:

website/docs/commands/state/rm.html.md

@@ -51,6 +51,11 @@ The command-line flags are all optional. The list of available flags are:
   Terraform-managed resources. By default it will use the configured backend,
   or the default "terraform.tfstate" if it exists.
 
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.
+
 ## Example: Remove a Resource
 
 The example below removes the `packet_device` resource named `worker`:

website/docs/commands/taint.html.markdown

@@ -65,6 +65,11 @@ The command-line flags are all optional. The list of available flags are:
   `-state` path will be used. Ignored when
   [remote state](/docs/state/remote.html) is used.
 
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.
+
 ## Example: Tainting a Single Resource
 
 This example will taint a single resource:

website/docs/commands/untaint.html.markdown

@@ -57,3 +57,8 @@ certain cases, see above note). The list of available flags are:
 * `-state-out=path` - Path to write updated state file. By default, the
   `-state` path will be used. Ignored when
   [remote state](/docs/state/remote.html) is used.
+
+* `-ignore-remote-version` - When using the enhanced remote backend with
+  Terraform Cloud, continue even if remote and local Terraform versions differ.
+  This may result in an unusable Terraform Cloud workspace, and should be used
+  with extreme caution.