Merge branch 'main' into fix-remote-backend-references

.circleci/config.yml

@@ -175,97 +175,6 @@ jobs:
           name: test docker build for 'full' image
           command: docker build -t test-docker-full .
 
-  # Based on a similar job in terraform-website repo.
-  website-link-check:
-    docker:
-      - image: *MIDDLEMAN_IMAGE
-    steps:
-      - checkout:
-          path: terraform
-
-      - run:
-          name: Determine changed website files, if any
-          working_directory: terraform
-          command: |
-            # Figure out what the current branch forked from. Compare against
-            # main and the set of "vX.Y" branches, and choose whichever branch
-            # we're the *fewest* commits ahead of.
-            # The point here isn't to perfectly predict where this will be
-            # merged; all we really care about is determining which commits are
-            # *unique to this PR,* so we don't accidentally complain about
-            # problems you had nothing to do with.
-            PARENT_BRANCH=$(
-              for br in $(git branch -rl --format='%(refname:short)' | grep -E '^origin/(main|v\d+\.\d+)$'); do
-                new_commits=$(git rev-list --first-parent ^${br} HEAD | wc -l);
-                echo "${br} ${new_commits}";
-              done \
-              | sort -n -k2 \
-              | head -n1 \
-              | awk '{print $1}';
-            )
-            echo "Checking current branch against: ${PARENT_BRANCH}"
-            MERGE_BASE=$(git merge-base HEAD ${PARENT_BRANCH})
-            git diff --name-only -z --diff-filter=AMRCT ${MERGE_BASE}..HEAD -- ./website/ > /tmp/changed-website-files.txt
-              # --name-only: Return a list of affected files but don't show the changes.
-              # -z: Make that a null-separated list (instead of newline-separated), and
-              #     DON'T mangle non-ASCII characters.
-              # --diff-filter=AMRCT: Only list files that were added, modified, renamed,
-              #     copied, or had their type changed (file, symlink, etc.). In
-              #     particular, we don't want to check deleted files.
-              # ${MERGE_BASE}..HEAD: Only consider files that have
-              #     changed since this branch diverged from its parent branch.
-              # -- ./website/: Only consider files in the website directory.
-            echo "Changed website files:"
-            cat /tmp/changed-website-files.txt | tr '\0' '\n'
-              # Need to use "tr" for display because it's a null-separated list.
-
-      - run:
-          name: Exit early if there's nothing to check
-          command: |
-            if [ ! -s /tmp/changed-website-files.txt ]; then
-              circleci-agent step halt
-            fi
-
-      - run:
-          name: Check out terraform-website repo
-          command: git clone git@github.com:hashicorp/terraform-website.git
-
-      - run:
-          name: Use local checkout for terraform submodule, instead of cloning again
-          working_directory: terraform-website
-          command: |
-            # Set submodule's URL to our existing checkout.
-            # (Using `pwd` because git's behavior with strictly relative paths is unreliable.)
-            git config --file=.gitmodules submodule.ext/terraform.url $(pwd)/../terraform/.git
-            # Make it so `make sync` will grab our current branch instead of stable-website.
-            git config --file=.gitmodules submodule.ext/terraform.branch HEAD
-
-      - run:
-          name: Init/update terraform-website submodules
-          working_directory: terraform-website
-          command: make sync
-
-      - run:
-          name: Set up terraform-website dependencies
-          working_directory: terraform-website/content
-          # If this does anything interesting, then the container needs an update.
-          command: bundle check || bundle install --path vendor/bundle --retry=3
-
-      - run:
-          name: Run middleman in background
-          working_directory: terraform-website/content
-          background: true
-          command: bundle exec middleman server
-
-      - run:
-          name: Wait for server to start
-          command: until curl -sS http://localhost:4567/ > /dev/null; do sleep 1; done
-
-      - run:
-          name: Check links in changed pages
-          working_directory: terraform-website/content
-          command: cat /tmp/changed-website-files.txt | bundle exec ./scripts/check-pr-links.rb
-
 workflows:
   version: 2
   test:
@@ -289,7 +198,3 @@ workflows:
       - build-amd64
       - build-arm
       - build-arm64
-
-  website-test:
-    jobs:
-      - website-link-check

.github/dependabot.yml

@@ -1,6 +0,0 @@
-version: 2
-updates:
-- package-ecosystem: gomod
-  directory: "/"
-  schedule:
-    interval: daily

.github/workflows/build-Dockerfile

@@ -0,0 +1,41 @@
+# This Dockerfile is not intended for general use, but is rather used to
+# produce our "light" release packages as part of our official release
+# pipeline.
+#
+# If you want to test this locally you'll need to set the three arguments
+# to values realistic for what the hashicorp/actions-docker-build GitHub
+# action would set, and ensure that there's a suitable "terraform" executable
+# in the dist/linux/${TARGETARCH} directory.
+
+FROM docker.mirror.hashicorp.services/alpine:latest AS default
+
+# This is intended to be run from the hashicorp/actions-docker-build GitHub
+# action, which sets these appropriately based on context.
+ARG PRODUCT_VERSION=UNSPECIFIED
+ARG PRODUCT_REVISION=UNSPECIFIED
+ARG BIN_NAME=terraform
+
+# This argument is set by the Docker toolchain itself, to the name
+# of the CPU architecture we're building an image for.
+# Our caller should've extracted the corresponding "terraform" executable
+# into dist/linux/${TARGETARCH} for us to use.
+ARG TARGETARCH
+
+LABEL maintainer="HashiCorp Terraform Team <terraform@hashicorp.com>"
+
+# New standard version label.
+LABEL version=$VERSION
+
+# Historical Terraform-specific label preserved for backward compatibility.
+LABEL "com.hashicorp.terraform.version"="${VERSION}"
+
+RUN apk add --no-cache git openssh
+
+# The hashicorp/actions-docker-build GitHub Action extracts the appropriate
+# release package for our target architecture into the current working
+# directory before running "docker build", which we'll then copy into the
+# Docker image to make sure that we use an identical binary as all of the
+# other official release channels.
+COPY ["dist/linux/${TARGETARCH}/terraform", "/bin/terraform"]
+
+ENTRYPOINT ["/bin/terraform"]

.github/workflows/build.yml

@@ -0,0 +1,480 @@
+name: Build Terraform CLI Packages
+
+# If you want to test changes to this file before merging to a main branch,
+# push them up to a branch whose name has the prefix "build-workflow-dev/",
+# which is a special prefix that triggers this workflow even though it's not
+# actually a release branch.
+
+# NOTE: This workflow is currently used only to verify that all commits to a
+# release branch are buildable. It's set up to generate some artifacts that
+# might in principle be consumed by a downstream release process, but currently
+# they are not used in this way and official Terraform CLI releases are instead
+# built using a separate process maintained elsewhere. We intend to adopt this
+# new process fully later, once other HashiCorp-internal tooling is ready.
+#
+# Currently this process produces what should be working packages but packages
+# NOT suitable for distribution to end-users as official releases, because it
+# doesn't include a step to ensure that "terraform version" (and similar) will
+# report the intended version number. Consequently we can safely use these
+# results for testing purposes, but not yet for release purposes. See the
+# "build" job below for a FIXME comment related to version numbers.
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - 'v[0-9]+.[0-9]+'
+      - build-workflow-dev/*
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+
+env:
+  PKG_NAME: "terraform"
+
+permissions:
+  contents: read
+  statuses: write
+
+jobs:
+  get-product-version:
+    name: "Determine intended Terraform version"
+    runs-on: ubuntu-latest
+    outputs:
+      product-version: ${{ steps.get-product-version.outputs.product-version }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Need all commits and tags to find a reasonable version number
+      - name: Git Describe
+        id: git-describe
+        run: |
+          git describe --first-parent
+          echo "::set-output name=raw-version::$(git describe --first-parent)"
+      - name: Decide version number
+        id: get-product-version
+        shell: bash
+        env:
+          RAW_VERSION: ${{ steps.git-describe.outputs.raw-version }}
+        run: |
+          echo "::set-output name=product-version::${RAW_VERSION#v}"
+      - name: Report chosen version number
+        run: |
+          [ -n "${{steps.get-product-version.outputs.product-version}}" ]
+          echo "::notice title=Terraform CLI Version::${{ steps.get-product-version.outputs.product-version }}"
+
+  get-go-version:
+    name: "Determine Go toolchain version"
+    runs-on: ubuntu-latest
+    outputs:
+      go-version: ${{ steps.get-go-version.outputs.go-version }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Determine Go version
+        id: get-go-version
+        # We use .go-version as our source of truth for current Go
+        # version, because "goenv" can react to it automatically.
+        run: |
+          echo "Building with Go $(cat .go-version)"
+          echo "::set-output name=go-version::$(cat .go-version)"
+
+  generate-metadata-file:
+    name: "Generate release metadata"
+    runs-on: ubuntu-latest
+    needs: get-product-version
+    outputs:
+      filepath: ${{ steps.generate-metadata-file.outputs.filepath }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Generate package metadata
+        id: generate-metadata-file
+        uses: hashicorp/actions-generate-metadata@main
+        with:
+          version: ${{ needs.get-product-version.outputs.product-version }}
+          product: ${{ env.PKG_NAME }}
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: metadata.json
+          path: ${{ steps.generate-metadata-file.outputs.filepath }}
+
+  build:
+    name: Build for ${{ matrix.goos }}_${{ matrix.goarch }}
+    runs-on: ${{ matrix.runson }}
+    needs:
+      - get-product-version
+      - get-go-version
+    strategy:
+      matrix:
+        include:
+          - {goos: "freebsd", goarch: "386", runson: "ubuntu-latest"}
+          - {goos: "freebsd", goarch: "amd64", runson: "ubuntu-latest"}
+          - {goos: "freebsd", goarch: "arm", runson: "ubuntu-latest"}
+          - {goos: "linux", goarch: "386", runson: "ubuntu-latest"}
+          - {goos: "linux", goarch: "amd64", runson: "ubuntu-latest"}
+          - {goos: "linux", goarch: "arm", runson: "ubuntu-latest"}
+          - {goos: "linux", goarch: "arm64", runson: "ubuntu-latest"}
+          - {goos: "openbsd", goarch: "386", runson: "ubuntu-latest"}
+          - {goos: "openbsd", goarch: "amd64", runson: "ubuntu-latest"}
+          - {goos: "solaris", goarch: "amd64", runson: "ubuntu-latest"}
+          - {goos: "windows", goarch: "386", runson: "ubuntu-latest"}
+          - {goos: "windows", goarch: "amd64", runson: "ubuntu-latest"}
+          - {goos: "darwin", goarch: "amd64", runson: "macos-latest"}
+          - {goos: "darwin", goarch: "arm64", runson: "macos-latest"}
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+
+      # FIXME: We're not currently setting the hard-coded version string in
+      # version/version.go at any point here, which means that the packages
+      # this process builds are not suitable for release. Once we're using
+      # Go 1.18 we may begin using the version information automatically
+      # embedded by the Go toolchain, at which point we won't need any
+      # special steps during build, but failing that we'll need to rework
+      # the version/version.go package so we can more readily update it
+      # using linker flags rather than direct code modification.
+
+      - name: Build
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+          ACTIONSOS: ${{ matrix.runson }}
+        run: |
+          mkdir dist out
+          if [ "$ACTIONSOS" == "macos-latest" ] && [ "$GOOS" == "darwin" ]; then
+            # When building for macOS _on_ macOS we must force CGo to get
+            # correct hostname resolution behavior. (This must be conditional
+            # because other cross-compiles won't have suitable headers
+            # available to use CGo; darwin_amd64 has suitable headers to
+            # cross-build for darwin_arm64.)
+            export CGO_ENABLED=1
+          fi
+          go build -ldflags "-w -s" -o dist/ .
+          zip -r -j out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip dist/
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          path: out/${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+
+  package-linux:
+    name: "Build Linux distro packages for ${{ matrix.arch }}"
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+      - build
+    strategy:
+      matrix:
+        include:
+          - {arch: "386"}
+          - {arch: "amd64"}
+          - {arch: "arm"}
+          - {arch: "arm64"}
+      fail-fast: false
+
+    env:
+      os: linux
+      arch: ${{matrix.arch}}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{ env.version }}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        run: |
+          mkdir -p dist
+          (cd dist && unzip "../terraform_${{ env.version }}_${{ env.os }}_${{ env.arch }}.zip")
+          mkdir -p out
+      - name: Build Linux distribution packages
+        uses: hashicorp/actions-packaging-linux@v1
+        with:
+          name: "terraform"
+          description: "Terraform enables you to safely and predictably create, change, and improve infrastructure. It is an open source tool that codifies APIs into declarative configuration files that can be shared amongst team members, treated as code, edited, reviewed, and versioned."
+          arch: ${{ matrix.arch }}
+          version: ${{ env.version }}
+          maintainer: "HashiCorp"
+          homepage: "https://terraform.io/"
+          license: "MPL-2.0"
+          binary: "dist/terraform"
+          deb_depends: "git"
+          rpm_depends: "git"
+      - name: Gather Linux distribution package filenames
+        run: |
+          echo "RPM_PACKAGE=$(basename out/*.rpm)" >> $GITHUB_ENV
+          echo "DEB_PACKAGE=$(basename out/*.deb)" >> $GITHUB_ENV
+      - name: "Save .rpm package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.RPM_PACKAGE }}
+          path: out/${{ env.RPM_PACKAGE }}
+      - name: "Save .deb package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.DEB_PACKAGE }}
+          path: out/${{ env.DEB_PACKAGE }}
+
+  # TODO: homebrew packages for macOS
+  #package-homebrew:
+  #  name: Build Homebrew package for darwin_${{ matrix.arch }}
+  #  runs-on: macos-latest
+  #  needs:
+  #    - get-product-version
+  #    - build
+  #  strategy:
+  #    matrix:
+  #      arch: ["amd64", "arm64"]
+  #    fail-fast: false
+  # ...
+
+  package-docker:
+    name: Build Docker image for linux_${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+      - build
+    strategy:
+      matrix:
+        arch: ["amd64"]
+      fail-fast: false
+
+    env:
+      repo: ${{github.event.repository.name}}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Docker images
+        uses: hashicorp/actions-docker-build@v1
+        with:
+          version: ${{env.version}}
+          target: default
+          arch: ${{matrix.arch}}
+          dockerfile: .github/workflows/build-Dockerfile
+          tags: |
+            docker.io/hashicorp/${{env.repo}}:${{env.version}}
+            986891699432.dkr.ecr.us-east-1.amazonaws.com/hashicorp/${{env.repo}}:${{env.version}}
+
+  e2etest-build:
+    name: Build e2etest for ${{ matrix.goos }}_${{ matrix.goarch }}
+    runs-on: ubuntu-latest
+    needs: ["get-go-version"]
+    strategy:
+      matrix:
+        # We build test harnesses only for the v1.0 Compatibility Promises
+        # supported platforms. Even within that set, we can only run on
+        # architectures for which we have GitHub Actions runners available,
+        # which is currently only amd64 (x64).
+        # TODO: GitHub Actions does support _self-hosted_ arm and arm64
+        # runners, so we could potentially run some ourselves to run our
+        # tests there, but at the time of writing there is no documented
+        # support for darwin_arm64 (macOS on Apple Silicon).
+        include:
+          - {goos: "darwin", goarch: "amd64"}
+          #- {goos: "darwin", goarch: "arm64"}
+          - {goos: "windows", goarch: "amd64"}
+          - {goos: "linux", goarch: "amd64"}
+          #- {goos: "linux", goarch: "arm"}
+          #- {goos: "linux", goarch: "arm64"}
+      fail-fast: false
+
+    env:
+      build_script: ./internal/command/e2etest/make-archive.sh
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Go toolchain
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ needs.get-go-version.outputs.go-version }}
+
+      - name: Build test harness package
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+        run: |
+          bash ./internal/command/e2etest/make-archive.sh
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          path: internal/command/e2etest/build/terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          if-no-files-found: error
+
+  e2etest-linux:
+    name: e2etest for linux_${{ matrix.goarch }}
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+      - build
+      - e2etest-build
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+          #- {goarch: "arm64"}
+          #- {goarch: "arm"}
+      fail-fast: false
+
+    env:
+      os: linux
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        run: |
+          unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip"
+          unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip"
+      - name: Run E2E Tests
+        run: |
+          TF_ACC=1 ./e2etest -test.v
+
+  e2etest-darwin:
+    name: e2etest for darwin_${{ matrix.goarch }}
+    runs-on: macos-latest
+    needs:
+      - get-product-version
+      - build
+      - e2etest-build
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+          #- {goarch: "arm64"}
+      fail-fast: false
+
+    env:
+      os: darwin
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        run: |
+          unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip"
+          unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip"
+      - name: Run E2E Tests
+        run: |
+          TF_ACC=1 ./e2etest -test.v
+
+  e2etest-windows:
+    name: e2etest for windows_${{ matrix.goarch }}
+    runs-on: windows-latest
+    needs:
+      - get-product-version
+      - build
+      - e2etest-build
+    strategy:
+      matrix:
+        include:
+          - {goarch: "amd64"}
+      fail-fast: false
+
+    env:
+      os: windows
+      arch: ${{ matrix.goarch }}
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      # NOTE: This intentionally _does not_ check out the source code
+      # for the commit/tag we're building, because by now we should
+      # have everything we need in the combination of CLI release package
+      # and e2etest package for this platform. (This helps ensure that we're
+      # really testing the release package and not inadvertently testing a
+      # fresh build from source.)
+      - name: "Download e2etest package"
+        uses: actions/download-artifact@v2
+        id: e2etestpkg
+        with:
+          name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: "Download Terraform CLI package"
+        uses: actions/download-artifact@v2
+        id: clipkg
+        with:
+          name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip
+          path: .
+      - name: Extract packages
+        shell: pwsh
+        run: |
+          Expand-Archive -LiteralPath 'terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.'
+          Expand-Archive -LiteralPath 'terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.'
+      - name: Run E2E Tests
+        env:
+          TF_ACC: 1
+        shell: cmd
+        run: |
+          e2etest.exe -test.v
+
+  docs-source-package:
+    name: "Build documentation bundle"
+    runs-on: ubuntu-latest
+    needs:
+      - get-product-version
+
+    env:
+      version: ${{needs.get-product-version.outputs.product-version}}
+
+    steps:
+      - uses: actions/checkout@v2
+      # FIXME: We should include some sort of pre-validation step here, to
+      # confirm that the doc content is mechanically valid so that the
+      # publishing pipeline will be able to render all content without errors.
+      - name: "Create documentation source bundle"
+        run: |
+          (cd website && zip -9 -r ../terraform-cli-docs-source_${{ env.version }}.zip .)
+      - uses: actions/upload-artifact@v2
+        with:
+          name: terraform-cli-docs-source_${{ env.version }}.zip
+          path: terraform-cli-docs-source_${{ env.version }}.zip
+          if-no-files-found: error

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## 1.2.0 (Unreleased)
 
+ENHANCEMENTS:
+
+* The "Invalid for_each argument" error message for unknown maps/sets now includes an additional paragraph to try to help the user notice they can move apply-time values into the map _values_ instead of the map _keys_, and thus avoid the problem without resorting to `-target`. [GH-30327]
+
+BUG FIXES:
+
+* Terraform now handles type constraints, nullability, and custom variable validation properly for root module variables. Previously there was an order of operations problem where the nullability and custom variable validation were checked too early, prior to dealing with the type constraints, and thus that logic could potentially "see" an incorrectly-typed value in spite of the type constraint, leading to incorrect errors. [GH-29959]
 
 ## Previous Releases
 

docs/plugin-protocol/releasing-new-version.md

@@ -41,7 +41,7 @@ Modify the `SupportedPluginProtocols` variable in hashicorp/terraform's
 
 Use the provider test framework to test a provider written with the new
 protocol. This end-to-end test ensures that providers written with the new
-protocol work correctly wtih the test framework, especially in communicating
+protocol work correctly with the test framework, especially in communicating
 the protocol version between the test framework and Terraform.
 
 ## Test Retrieving and Running a Provider From the Registry

docs/plugin-protocol/tfplugin6.2.proto

@@ -0,0 +1,350 @@
+// Terraform Plugin RPC protocol version 6.2
+//
+// This file defines version 6.2 of the RPC protocol. To implement a plugin
+// against this protocol, copy this definition into your own codebase and
+// use protoc to generate stubs for your target language.
+//
+// This file will not be updated. Any minor versions of protocol 6 to follow
+// should copy this file and modify the copy while maintaing backwards
+// compatibility. Breaking changes, if any are required, will come
+// in a subsequent major version with its own separate proto definition.
+//
+// Note that only the proto files included in a release tag of Terraform are
+// official protocol releases. Proto files taken from other commits may include
+// incomplete changes or features that did not make it into a final release.
+// In all reasonable cases, plugin developers should take the proto file from
+// the tag of the most recent release of Terraform, and not from the main
+// branch or any other development branch.
+//
+syntax = "proto3";
+option go_package = "github.com/hashicorp/terraform/internal/tfplugin6";
+
+package tfplugin6;
+
+// DynamicValue is an opaque encoding of terraform data, with the field name
+// indicating the encoding scheme used.
+message DynamicValue {
+    bytes msgpack = 1;
+    bytes json = 2;
+}
+
+message Diagnostic {
+    enum Severity {
+        INVALID = 0;
+        ERROR = 1;
+        WARNING = 2;
+    }
+    Severity severity = 1;
+    string summary = 2;
+    string detail = 3;
+    AttributePath attribute = 4;
+}
+
+message AttributePath {
+    message Step {
+        oneof selector {
+            // Set "attribute_name" to represent looking up an attribute
+            // in the current object value.
+            string attribute_name = 1;
+            // Set "element_key_*" to represent looking up an element in
+            // an indexable collection type.
+            string element_key_string = 2;
+            int64 element_key_int = 3;
+        }
+    }
+    repeated Step steps = 1;
+}
+
+message StopProvider {
+    message Request {
+    }
+    message Response {
+        string Error = 1;
+    }
+}
+
+// RawState holds the stored state for a resource to be upgraded by the
+// provider. It can be in one of two formats, the current json encoded format
+// in bytes, or the legacy flatmap format as a map of strings.
+message RawState {
+    bytes json = 1;
+    map<string, string> flatmap = 2;
+}
+
+enum StringKind {
+    PLAIN = 0;
+    MARKDOWN = 1;
+}
+
+// Schema is the configuration schema for a Resource or Provider.
+message Schema {
+    message Block {
+        int64 version = 1;
+        repeated Attribute attributes = 2;
+        repeated NestedBlock block_types = 3;
+        string description = 4;
+        StringKind description_kind = 5;
+        bool deprecated = 6;
+    }
+
+    message Attribute {
+        string name = 1;
+        bytes type = 2;
+        Object nested_type = 10;
+        string description = 3;
+        bool required = 4;
+        bool optional = 5;
+        bool computed = 6;
+        bool sensitive = 7;
+        StringKind description_kind = 8;
+        bool deprecated = 9;
+    }
+
+    message NestedBlock {
+        enum NestingMode {
+            INVALID = 0;
+            SINGLE = 1;
+            LIST = 2;
+            SET = 3;
+            MAP = 4;
+            GROUP = 5;
+        }
+
+        string type_name = 1;
+        Block block = 2;
+        NestingMode nesting = 3;
+        int64 min_items = 4;
+        int64 max_items = 5;
+    }
+
+    message Object {
+        enum NestingMode {
+            INVALID = 0;
+            SINGLE = 1;
+            LIST = 2;
+            SET = 3;
+            MAP = 4;
+        }
+
+        repeated Attribute attributes = 1;
+        NestingMode nesting = 3;
+
+        // MinItems and MaxItems were never used in the protocol, and have no
+        // effect on validation.
+        int64 min_items = 4 [deprecated = true];
+        int64 max_items = 5 [deprecated = true];
+    }
+
+    // The version of the schema.
+    // Schemas are versioned, so that providers can upgrade a saved resource
+    // state when the schema is changed.
+    int64 version = 1;
+
+    // Block is the top level configuration block for this schema.
+    Block block = 2;
+}
+
+service Provider {
+    //////// Information about what a provider supports/expects
+    rpc GetProviderSchema(GetProviderSchema.Request) returns (GetProviderSchema.Response);
+    rpc ValidateProviderConfig(ValidateProviderConfig.Request) returns (ValidateProviderConfig.Response);
+    rpc ValidateResourceConfig(ValidateResourceConfig.Request) returns (ValidateResourceConfig.Response);
+    rpc ValidateDataResourceConfig(ValidateDataResourceConfig.Request) returns (ValidateDataResourceConfig.Response);
+    rpc UpgradeResourceState(UpgradeResourceState.Request) returns (UpgradeResourceState.Response);
+
+    //////// One-time initialization, called before other functions below
+    rpc ConfigureProvider(ConfigureProvider.Request) returns (ConfigureProvider.Response);
+
+    //////// Managed Resource Lifecycle
+    rpc ReadResource(ReadResource.Request) returns (ReadResource.Response);
+    rpc PlanResourceChange(PlanResourceChange.Request) returns (PlanResourceChange.Response);
+    rpc ApplyResourceChange(ApplyResourceChange.Request) returns (ApplyResourceChange.Response);
+    rpc ImportResourceState(ImportResourceState.Request) returns (ImportResourceState.Response);
+
+    rpc ReadDataSource(ReadDataSource.Request) returns (ReadDataSource.Response);
+
+    //////// Graceful Shutdown
+    rpc StopProvider(StopProvider.Request) returns (StopProvider.Response);
+}
+
+message GetProviderSchema {
+    message Request {
+    }
+    message Response {
+        Schema provider = 1;
+        map<string, Schema> resource_schemas = 2;
+        map<string, Schema> data_source_schemas = 3;
+        repeated Diagnostic diagnostics = 4;
+        Schema provider_meta = 5;
+    }
+}
+
+message ValidateProviderConfig {
+    message Request {
+        DynamicValue config = 1;
+    }
+    message Response {
+        repeated Diagnostic diagnostics = 2;
+    }
+}
+
+message UpgradeResourceState {
+    message Request {
+        string type_name = 1;
+
+        // version is the schema_version number recorded in the state file
+        int64 version = 2;
+
+        // raw_state is the raw states as stored for the resource.  Core does
+        // not have access to the schema of prior_version, so it's the
+        // provider's responsibility to interpret this value using the
+        // appropriate older schema. The raw_state will be the json encoded
+        // state, or a legacy flat-mapped format.
+        RawState raw_state = 3;
+    }
+    message Response {
+        // new_state is a msgpack-encoded data structure that, when interpreted with
+        // the _current_ schema for this resource type, is functionally equivalent to
+        // that which was given in prior_state_raw.
+        DynamicValue upgraded_state = 1;
+
+        // diagnostics describes any errors encountered during migration that could not
+        // be safely resolved, and warnings about any possibly-risky assumptions made
+        // in the upgrade process.
+        repeated Diagnostic diagnostics = 2;
+    }
+}
+
+message ValidateResourceConfig {
+    message Request {
+        string type_name = 1;
+        DynamicValue config = 2;
+    }
+    message Response {
+        repeated Diagnostic diagnostics = 1;
+    }
+}
+
+message ValidateDataResourceConfig {
+    message Request {
+        string type_name = 1;
+        DynamicValue config = 2;
+    }
+    message Response {
+        repeated Diagnostic diagnostics = 1;
+    }
+}
+
+message ConfigureProvider {
+    message Request {
+        string terraform_version = 1;
+        DynamicValue config = 2;
+    }
+    message Response {
+        repeated Diagnostic diagnostics = 1;
+    }
+}
+
+message ReadResource {
+    message Request {
+        string type_name = 1;
+        DynamicValue current_state = 2;
+        bytes private = 3;
+        DynamicValue provider_meta = 4;
+    }
+    message Response {
+        DynamicValue new_state = 1;
+        repeated Diagnostic diagnostics = 2;
+        bytes private = 3;
+    }
+}
+
+message PlanResourceChange {
+    message Request {
+        string type_name = 1;
+        DynamicValue prior_state = 2;
+        DynamicValue proposed_new_state = 3;
+        DynamicValue config = 4;
+        bytes prior_private = 5;
+        DynamicValue provider_meta = 6;
+    }
+
+    message Response {
+        DynamicValue planned_state = 1;
+        repeated AttributePath requires_replace = 2;
+        bytes planned_private = 3;
+        repeated Diagnostic diagnostics = 4;
+
+        // This may be set only by the helper/schema "SDK" in the main Terraform
+        // repository, to request that Terraform Core >=0.12 permit additional
+        // inconsistencies that can result from the legacy SDK type system
+        // and its imprecise mapping to the >=0.12 type system.
+        // The change in behavior implied by this flag makes sense only for the
+        // specific details of the legacy SDK type system, and are not a general
+        // mechanism to avoid proper type handling in providers.
+        //
+        //     ====              DO NOT USE THIS              ====
+        //     ==== THIS MUST BE LEFT UNSET IN ALL OTHER SDKS ====
+        //     ====              DO NOT USE THIS              ====
+        bool legacy_type_system = 5;
+    }
+}
+
+message ApplyResourceChange {
+    message Request {
+        string type_name = 1;
+        DynamicValue prior_state = 2;
+        DynamicValue planned_state = 3;
+        DynamicValue config = 4;
+        bytes planned_private = 5;
+        DynamicValue provider_meta = 6;
+    }
+    message Response {
+        DynamicValue new_state = 1;
+        bytes private = 2;
+        repeated Diagnostic diagnostics = 3;
+
+        // This may be set only by the helper/schema "SDK" in the main Terraform
+        // repository, to request that Terraform Core >=0.12 permit additional
+        // inconsistencies that can result from the legacy SDK type system
+        // and its imprecise mapping to the >=0.12 type system.
+        // The change in behavior implied by this flag makes sense only for the
+        // specific details of the legacy SDK type system, and are not a general
+        // mechanism to avoid proper type handling in providers.
+        //
+        //     ====              DO NOT USE THIS              ====
+        //     ==== THIS MUST BE LEFT UNSET IN ALL OTHER SDKS ====
+        //     ====              DO NOT USE THIS              ====
+        bool legacy_type_system = 4;
+    }
+}
+
+message ImportResourceState {
+    message Request {
+        string type_name = 1;
+        string id = 2;
+    }
+
+    message ImportedResource {
+        string type_name = 1;
+        DynamicValue state = 2;
+        bytes private = 3;
+    }
+
+    message Response {
+        repeated ImportedResource imported_resources = 1;
+        repeated Diagnostic diagnostics = 2;
+    }
+}
+
+message ReadDataSource {
+    message Request {
+        string type_name = 1;
+        DynamicValue config = 2;
+        DynamicValue provider_meta = 3;
+    }
+    message Response {
+        DynamicValue state = 1;
+        repeated Diagnostic diagnostics = 2;
+    }
+}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/apparentlymart/go-userdirs v0.0.0-20200915174352-b0c018a67c13
 	github.com/apparentlymart/go-versions v1.0.1
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
-	github.com/aws/aws-sdk-go v1.40.25
+	github.com/aws/aws-sdk-go v1.42.35
 	github.com/bgentry/speakeasy v0.1.0
 	github.com/bmatcuk/doublestar v1.1.5
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
@@ -36,12 +36,11 @@ require (
 	github.com/hashicorp/go-azure-helpers v0.18.0
 	github.com/hashicorp/go-checkpoint v0.5.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
-	github.com/hashicorp/go-getter v1.5.9
+	github.com/hashicorp/go-getter v1.5.10
 	github.com/hashicorp/go-hclog v0.15.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-plugin v1.4.3
 	github.com/hashicorp/go-retryablehttp v0.7.0
-	github.com/hashicorp/go-safetemp v1.0.0
 	github.com/hashicorp/go-tfe v0.21.0
 	github.com/hashicorp/go-uuid v1.0.2
 	github.com/hashicorp/go-version v1.3.0
@@ -85,7 +84,7 @@ require (
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20210428180535-15715dcf1ace
 	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
 	golang.org/x/mod v0.4.2
-	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d
+	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
 	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d
@@ -145,6 +144,7 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.4 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-safetemp v1.0.0 // indirect
 	github.com/hashicorp/go-slug v0.7.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/hashicorp/jsonapi v0.0.0-20210826224640-ee7dae0fb22d // indirect

go.sum

@@ -151,8 +151,8 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
 github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.40.25 h1:Depnx7O86HWgOCLD5nMto6F9Ju85Q1QuFDnbpZYQWno=
+github.com/aws/aws-sdk-go v1.42.35 h1:N4N9buNs4YlosI9N0+WYrq8cIZwdgv34yRbxzZlTvFs=
-github.com/aws/aws-sdk-go v1.40.25/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
+github.com/aws/aws-sdk-go v1.42.35/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -375,8 +375,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320/go.mod h1:EiZBMaudVLy8fmjf9Npq1dq9RalhveqZG5w/yz3mHWs=
 github.com/hashicorp/go-getter v1.5.3/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI=
-github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0=
+github.com/hashicorp/go-getter v1.5.10 h1:EN9YigTlv5Ola0IuleFzQGuaYPPHHtWusP/5AypWEMs=
-github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI=
+github.com/hashicorp/go-getter v1.5.10/go.mod h1:9i48BP6wpWweI/0/+FBjqLrp9S8XtwUGjiu0QkWHEaY=
 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI=
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
@@ -839,9 +839,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
+golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=

internal/addrs/module_instance_test.go

@@ -162,9 +162,9 @@ func TestModuleInstance_IsDeclaredByCall(t *testing.T) {
 }
 
 func mustParseModuleInstanceStr(str string) ModuleInstance {
-	mi, err := ParseModuleInstanceStr(str)
+	mi, diags := ParseModuleInstanceStr(str)
-	if err != nil {
+	if diags.HasErrors() {
-		panic(err)
+		panic(diags.ErrWithWarnings())
 	}
 	return mi
 }

internal/addrs/move_endpoint_module.go

@@ -373,7 +373,7 @@ func (e *MoveEndpointInModule) CanChainFrom(other *MoveEndpointInModule) bool {
 	return false
 }
 
-// NestedWithin returns true if the reciever describes an address that is
+// NestedWithin returns true if the receiver describes an address that is
 // contained within one of the objects that the given other address could
 // select.
 func (e *MoveEndpointInModule) NestedWithin(other *MoveEndpointInModule) bool {
@@ -704,3 +704,37 @@ func (r AbsResourceInstance) MoveDestination(fromMatch, toMatch *MoveEndpointInM
 		panic("unexpected object kind")
 	}
 }
+
+// IsModuleReIndex takes the From and To endpoints from a single move
+// statement, and returns true if the only changes are to module indexes, and
+// all non-absolute paths remain the same.
+func (from *MoveEndpointInModule) IsModuleReIndex(to *MoveEndpointInModule) bool {
+	// The statements must originate from the same module.
+	if !from.module.Equal(to.module) {
+		panic("cannot compare move expressions from different modules")
+	}
+
+	switch f := from.relSubject.(type) {
+	case AbsModuleCall:
+		switch t := to.relSubject.(type) {
+		case ModuleInstance:
+			// Generate a synthetic module to represent the full address of
+			// the module call. We're not actually comparing indexes, so the
+			// instance doesn't matter.
+			callAddr := f.Instance(NoKey).Module()
+			return callAddr.Equal(t.Module())
+		}
+
+	case ModuleInstance:
+		switch t := to.relSubject.(type) {
+		case AbsModuleCall:
+			callAddr := t.Instance(NoKey).Module()
+			return callAddr.Equal(f.Module())
+
+		case ModuleInstance:
+			return t.Module().Equal(f.Module())
+		}
+	}
+
+	return false
+}

internal/addrs/move_endpoint_module_test.go

@@ -1584,6 +1584,158 @@ func TestSelectsResource(t *testing.T) {
 	}
 }
 
+func TestIsModuleMoveReIndex(t *testing.T) {
+	tests := []struct {
+		from, to AbsMoveable
+		expect   bool
+	}{
+		{
+			from:   mustParseModuleInstanceStr(`module.bar`),
+			to:     mustParseModuleInstanceStr(`module.bar`),
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar`),
+			to:     mustParseModuleInstanceStr(`module.bar[0]`),
+			expect: true,
+		},
+		{
+			from: AbsModuleCall{
+				Call: ModuleCall{Name: "bar"},
+			},
+			to:     mustParseModuleInstanceStr(`module.bar[0]`),
+			expect: true,
+		},
+		{
+			from: mustParseModuleInstanceStr(`module.bar["a"]`),
+			to: AbsModuleCall{
+				Call: ModuleCall{Name: "bar"},
+			},
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.foo`),
+			to:     mustParseModuleInstanceStr(`module.bar`),
+			expect: false,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar`),
+			to:     mustParseModuleInstanceStr(`module.foo[0]`),
+			expect: false,
+		},
+		{
+			from: AbsModuleCall{
+				Call: ModuleCall{Name: "bar"},
+			},
+			to:     mustParseModuleInstanceStr(`module.foo[0]`),
+			expect: false,
+		},
+		{
+			from: mustParseModuleInstanceStr(`module.bar["a"]`),
+			to: AbsModuleCall{
+				Call: ModuleCall{Name: "foo"},
+			},
+			expect: false,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz`),
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz`),
+			to:     mustParseModuleInstanceStr(`module.baz.module.baz`),
+			expect: false,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz`),
+			to:     mustParseModuleInstanceStr(`module.baz.module.baz[0]`),
+			expect: false,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar[0].module.baz`),
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar[0].module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			expect: true,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar[0].module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar[1].module.baz[0]`),
+			expect: true,
+		},
+		{
+			from: AbsModuleCall{
+				Call: ModuleCall{Name: "baz"},
+			},
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			expect: false,
+		},
+		{
+			from: mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			to: AbsModuleCall{
+				Call: ModuleCall{Name: "baz"},
+			},
+			expect: false,
+		},
+
+		{
+			from: AbsModuleCall{
+				Module: mustParseModuleInstanceStr(`module.bar[0]`),
+				Call:   ModuleCall{Name: "baz"},
+			},
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			expect: true,
+		},
+
+		{
+			from: mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			to: AbsModuleCall{
+				Module: mustParseModuleInstanceStr(`module.bar[0]`),
+				Call:   ModuleCall{Name: "baz"},
+			},
+			expect: true,
+		},
+
+		{
+			from:   mustParseModuleInstanceStr(`module.baz`),
+			to:     mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			expect: false,
+		},
+		{
+			from:   mustParseModuleInstanceStr(`module.bar.module.baz[0]`),
+			to:     mustParseModuleInstanceStr(`module.baz`),
+			expect: false,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("[%02d]IsModuleMoveReIndex(%s, %s)", i, test.from, test.to),
+			func(t *testing.T) {
+				from := &MoveEndpointInModule{
+					relSubject: test.from,
+				}
+
+				to := &MoveEndpointInModule{
+					relSubject: test.to,
+				}
+
+				if got := from.IsModuleReIndex(to); got != test.expect {
+					t.Errorf("expected %t, got %t", test.expect, got)
+				}
+			},
+		)
+	}
+}
+
 func mustParseAbsResourceInstanceStr(s string) AbsResourceInstance {
 	r, diags := ParseAbsResourceInstanceStr(s)
 	if diags.HasErrors() {

internal/backend/unparsed_value.go

@@ -164,13 +164,18 @@ func ParseVariableValues(vv map[string]UnparsedVariableValue, decls map[string]*
 
 	// By this point we should've gathered all of the required root module
 	// variables from one of the many possible sources. We'll now populate
-	// any we haven't gathered as their defaults and fail if any of the
+	// any we haven't gathered as unset placeholders which Terraform Core
-	// missing ones are required.
+	// can then react to.
 	for name, vc := range decls {
 		if isDefinedAny(name, ret, undeclared) {
 			continue
 		}
 
+		// This check is redundant with a check made in Terraform Core when
+		// processing undeclared variables, but allows us to generate a more
+		// specific error message which mentions -var and -var-file command
+		// line options, whereas the one in Terraform Core is more general
+		// due to supporting both root and child module variables.
 		if vc.Required() {
 			diags = diags.Append(&hcl.Diagnostic{
 				Severity: hcl.DiagError,
@@ -189,8 +194,14 @@ func ParseVariableValues(vv map[string]UnparsedVariableValue, decls map[string]*
 				SourceRange: tfdiags.SourceRangeFromHCL(vc.DeclRange),
 			}
 		} else {
+			// We're still required to put an entry for this variable
+			// in the mapping to be explicit to Terraform Core that we
+			// visited it, but its value will be cty.NilVal to represent
+			// that it wasn't set at all at this layer, and so Terraform Core
+			// should substitute a default if available, or generate an error
+			// if not.
 			ret[name] = &terraform.InputValue{
-				Value:       vc.Default,
+				Value:       cty.NilVal,
 				SourceType:  terraform.ValueFromConfig,
 				SourceRange: tfdiags.SourceRangeFromHCL(vc.DeclRange),
 			}

internal/backend/unparsed_value_test.go

@@ -204,7 +204,7 @@ func TestUnparsedValue(t *testing.T) {
 				},
 			},
 			"missing2": {
-				Value:      cty.StringVal("default for missing2"),
+				Value:      cty.NilVal, // Terraform Core handles substituting the default
 				SourceType: terraform.ValueFromConfig,
 				SourceRange: tfdiags.SourceRange{
 					Filename: "fake.tf",

internal/cloud/e2e/apply_auto_approve_test.go

@@ -2,26 +2,20 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
 func Test_terraform_apply_autoApprove(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
 
-	cases := map[string]struct {
+	cases := testCases{
-		operations  []operationSets
-		validations func(t *testing.T, orgName string)
-	}{
 		"workspace manual apply, terraform apply without auto-approve, expect prompt": {
 			operations: []operationSets{
 				{
@@ -179,76 +173,6 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
 			},
 		},
 	}
-	for name, tc := range cases {
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
 
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
+	testRunner(t, cases, 1)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, organization.Name)
-			}
-		})
-	}
 }

internal/cloud/e2e/backend_apply_before_init_test.go

@@ -1,22 +1,15 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
-
-	expect "github.com/Netflix/go-expect"
-	"github.com/hashicorp/terraform/internal/e2e"
 )
 
 func Test_backend_apply_before_init(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
-	// t.Parallel()
 	skipWithoutRemoteTerraformVersion(t)
 
-	cases := map[string]struct {
+	cases := testCases{
-		operations []operationSets
-	}{
 		"terraform apply with cloud block - blank state": {
 			operations: []operationSets{
 				{
@@ -71,72 +64,5 @@ func Test_backend_apply_before_init(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-			}
-		})
-	}
 }

internal/cloud/e2e/helper_test.go

@@ -15,7 +15,9 @@ import (
 )
 
 const (
-	expectConsoleTimeout = 15 * time.Second
+	// We need to give the console enough time to hear back.
+	// 1 minute was too short in some cases, so this gives it ample time.
+	expectConsoleTimeout = 3 * time.Minute
 )
 
 type tfCommand struct {

internal/cloud/e2e/init_with_empty_tags_test.go

@@ -1,22 +1,15 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
-
-	expect "github.com/Netflix/go-expect"
-	"github.com/hashicorp/terraform/internal/e2e"
 )
 
 func Test_init_with_empty_tags(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
-	// t.Parallel()
 	skipWithoutRemoteTerraformVersion(t)
 
-	cases := map[string]struct {
+	cases := testCases{
-		operations []operationSets
-	}{
 		"terraform init with cloud block - no tagged workspaces exist yet": {
 			operations: []operationSets{
 				{
@@ -38,71 +31,5 @@ func Test_init_with_empty_tags(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-			}
-		})
-	}
 }

internal/cloud/e2e/main_test.go

@@ -10,7 +10,9 @@ import (
 	"strings"
 	"testing"
 
+	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
+	"github.com/hashicorp/terraform/internal/e2e"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
@@ -66,6 +68,96 @@ func setup() func() {
 		teardown()
 	}
 }
+func testRunner(t *testing.T, cases testCases, orgCount int, tfEnvFlags ...string) {
+	for name, tc := range cases {
+		tc := tc // rebind tc into this lexical scope
+		t.Run(name, func(subtest *testing.T) {
+			subtest.Parallel()
+
+			orgNames := []string{}
+			for i := 0; i < orgCount; i++ {
+				organization, cleanup := createOrganization(t)
+				t.Cleanup(cleanup)
+				orgNames = append(orgNames, organization.Name)
+			}
+
+			exp, err := expect.NewConsole(defaultOpts()...)
+			if err != nil {
+				subtest.Fatal(err)
+			}
+			defer exp.Close()
+
+			tmpDir, err := ioutil.TempDir("", "terraform-test")
+			if err != nil {
+				subtest.Fatal(err)
+			}
+			defer os.RemoveAll(tmpDir)
+
+			tf := e2e.NewBinary(terraformBin, tmpDir)
+			tfEnvFlags = append(tfEnvFlags, "TF_LOG=INFO")
+			tfEnvFlags = append(tfEnvFlags, cliConfigFileEnv)
+			for _, env := range tfEnvFlags {
+				tf.AddEnv(env)
+			}
+			defer tf.Close()
+
+			var orgName string
+			for index, op := range tc.operations {
+				if orgCount == 1 {
+					orgName = orgNames[0]
+				} else {
+					orgName = orgNames[index]
+				}
+				op.prep(t, orgName, tf.WorkDir())
+				for _, tfCmd := range op.commands {
+					cmd := tf.Cmd(tfCmd.command...)
+					cmd.Stdin = exp.Tty()
+					cmd.Stdout = exp.Tty()
+					cmd.Stderr = exp.Tty()
+
+					err = cmd.Start()
+					if err != nil {
+						subtest.Fatal(err)
+					}
+
+					if tfCmd.expectedCmdOutput != "" {
+						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
+						if err != nil {
+							subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
+						}
+					}
+
+					lenInput := len(tfCmd.userInput)
+					lenInputOutput := len(tfCmd.postInputOutput)
+					if lenInput > 0 {
+						for i := 0; i < lenInput; i++ {
+							input := tfCmd.userInput[i]
+							exp.SendLine(input)
+							// use the index to find the corresponding
+							// output that matches the input.
+							if lenInputOutput-1 >= i {
+								output := tfCmd.postInputOutput[i]
+								_, err := exp.ExpectString(output)
+								if err != nil {
+									subtest.Fatal(err)
+								}
+							}
+						}
+					}
+
+					err = cmd.Wait()
+					if err != nil && !tfCmd.expectError {
+						subtest.Fatal(err)
+					}
+				}
+			}
+
+			if tc.validations != nil {
+				tc.validations(t, orgName)
+			}
+		})
+	}
+}
 
 func setTfeClient() {
 	tfeHostname = os.Getenv("TFE_HOSTNAME")

internal/cloud/e2e/migrate_state_multi_to_tfc_test.go

@@ -2,26 +2,20 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
 func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
 
-	cases := map[string]struct {
+	cases := testCases{
-		operations  []operationSets
-		validations func(t *testing.T, orgName string)
-	}{
 		"migrating multiple workspaces to cloud using name strategy; current workspace is 'default'": {
 			operations: []operationSets{
 				{
@@ -224,81 +218,11 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			defer tf.Close()
-			tf.AddEnv(cliConfigFileEnv)
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, organization.Name)
-			}
-		})
-	}
 }
 
 func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
@@ -512,79 +436,5 @@ func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			defer tf.Close()
-			tf.AddEnv(cliConfigFileEnv)
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								if output == "" {
-									continue
-								}
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil {
-						t.Fatal(err)
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, organization.Name)
-			}
-		})
-	}
 }

internal/cloud/e2e/migrate_state_remote_backend_to_tfc_test.go

@@ -2,21 +2,20 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 )
 
-func Test_migrate_remote_backend_name_to_tfc_name(t *testing.T) {
+func Test_migrate_remote_backend_single_org(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
-	operations := []operationSets{
+	cases := testCases{
+		"migrate remote backend name to tfc name": {
+			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						remoteWorkspace := "remote-workspace"
@@ -55,8 +54,8 @@ func Test_migrate_remote_backend_name_to_tfc_name(t *testing.T) {
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				expectedName := "cloud-workspace"
 				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
 				if err != nil {
@@ -65,81 +64,10 @@ func Test_migrate_remote_backend_name_to_tfc_name(t *testing.T) {
 				if ws == nil {
 					t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
 				}
-	}
+			},
-
+		},
-	exp, err := expect.NewConsole(defaultOpts()...)
+		"migrate remote backend name to tfc same name": {
-	if err != nil {
+			operations: []operationSets{
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_name_to_tfc_same_name(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						remoteWorkspace := "remote-workspace"
@@ -178,8 +106,8 @@ func Test_migrate_remote_backend_name_to_tfc_same_name(t *testing.T) {
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				expectedName := "remote-workspace"
 				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
 				if err != nil {
@@ -188,211 +116,10 @@ func Test_migrate_remote_backend_name_to_tfc_same_name(t *testing.T) {
 				if ws == nil {
 					t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
 				}
-	}
-
-	exp, err := expect.NewConsole(defaultOpts()...)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_name_to_tfc_name_different_org(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
-		{
-			prep: func(t *testing.T, orgName, dir string) {
-				remoteWorkspace := "remote-workspace"
-				tfBlock := terraformConfigRemoteBackendName(orgName, remoteWorkspace)
-				writeMainTF(t, tfBlock, dir)
-			},
-			commands: []tfCommand{
-				{
-					command:           []string{"init"},
-					expectedCmdOutput: `Successfully configured the backend "remote"!`,
-				},
-				{
-					command:         []string{"apply", "-auto-approve"},
-					postInputOutput: []string{`Apply complete!`},
 			},
 		},
-		},
+		"migrate remote backend name to tfc tags": {
-		{
+			operations: []operationSets{
-			prep: func(t *testing.T, orgName, dir string) {
-				wsName := "remote-workspace"
-				tfBlock := terraformConfigCloudBackendName(orgName, wsName)
-				writeMainTF(t, tfBlock, dir)
-			},
-			commands: []tfCommand{
-				{
-					command:           []string{"init", "-ignore-remote-version"},
-					expectedCmdOutput: `Migrating from backend "remote" to Terraform Cloud.`,
-					userInput:         []string{"yes", "yes"},
-					postInputOutput: []string{
-						`Should Terraform migrate your existing state?`,
-						`Terraform Cloud has been successfully initialized!`},
-				},
-				{
-					command:           []string{"workspace", "show"},
-					expectedCmdOutput: `remote-workspace`,
-				},
-			},
-		},
-	}
-	validations := func(t *testing.T, orgName string) {
-		expectedName := "remote-workspace"
-		ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if ws == nil {
-			t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
-		}
-	}
-
-	exp, err := expect.NewConsole(defaultOpts()...)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	orgOne, cleanupOne := createOrganization(t)
-	orgTwo, cleanupTwo := createOrganization(t)
-	defer cleanupOne()
-	defer cleanupTwo()
-	orgs := []string{orgOne.Name, orgTwo.Name}
-	var orgName string
-	for index, op := range operations {
-		orgName = orgs[index]
-		op.prep(t, orgName, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, orgName)
-	}
-}
-
-func Test_migrate_remote_backend_name_to_tfc_tags(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						remoteWorkspace := "remote-workspace"
@@ -436,8 +163,8 @@ func Test_migrate_remote_backend_name_to_tfc_tags(t *testing.T) {
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				wsList, err := tfeClient.Workspaces.List(ctx, orgName, tfe.WorkspaceListOptions{
 					Tags: tfe.String("app"),
 				})
@@ -451,82 +178,10 @@ func Test_migrate_remote_backend_name_to_tfc_tags(t *testing.T) {
 				if ws.Name != "cloud-workspace" {
 					t.Fatalf("Expected workspace to be `cloud-workspace`, but is %s", ws.Name)
 				}
-	}
+			},
-
+		},
-	exp, err := expect.NewConsole(defaultOpts()...)
+		"migrate remote backend prefix to tfc name strategy single workspace": {
-	if err != nil {
+			operations: []operationSets{
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_single_workspace(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						_ = createWorkspace(t, orgName, tfe.WorkspaceCreateOptions{Name: tfe.String("app-one")})
@@ -566,8 +221,8 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_single_workspace(t
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				expectedName := "cloud-workspace"
 				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
 				if err != nil {
@@ -576,82 +231,10 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_single_workspace(t
 				if ws == nil {
 					t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
 				}
-	}
+			},
-
+		},
-	exp, err := expect.NewConsole(defaultOpts()...)
+		"migrate remote backend prefix to tfc name strategy multi workspace": {
-	if err != nil {
+			operations: []operationSets{
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						got, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", output, err, got)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_multi_workspace(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						_ = createWorkspace(t, orgName, tfe.WorkspaceCreateOptions{Name: tfe.String("app-one")})
@@ -701,8 +284,8 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_multi_workspace(t *
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				expectedName := "cloud-workspace"
 				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
 				if err != nil {
@@ -730,82 +313,10 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_multi_workspace(t *
 				if empty {
 					t.Fatalf("expected workspaces to include 'app-two' but didn't.")
 				}
-	}
+			},
-
+		},
-	exp, err := expect.NewConsole(defaultOpts()...)
+		"migrate remote backend prefix to tfc tags strategy single workspace": {
-	if err != nil {
+			operations: []operationSets{
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_single_workspace(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						_ = createWorkspace(t, orgName, tfe.WorkspaceCreateOptions{Name: tfe.String("app-one")})
@@ -846,8 +357,8 @@ func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_single_workspace(t
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				expectedName := "cloud-workspace"
 				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
 				if err != nil {
@@ -856,82 +367,10 @@ func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_single_workspace(t
 				if ws == nil {
 					t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
 				}
-	}
+			},
-
+		},
-	exp, err := expect.NewConsole(defaultOpts()...)
+		"migrate remote backend prefix to tfc tags strategy multi workspace": {
-	if err != nil {
+			operations: []operationSets{
-		t.Fatal(err)
-	}
-	defer exp.Close()
-
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	tf := e2e.NewBinary(terraformBin, tmpDir)
-	tf.AddEnv(cliConfigFileEnv)
-	defer tf.Close()
-
-	organization, cleanup := createOrganization(t)
-	defer cleanup()
-	for _, op := range operations {
-		op.prep(t, organization.Name, tf.WorkDir())
-		for _, tfCmd := range op.commands {
-			cmd := tf.Cmd(tfCmd.command...)
-			cmd.Stdin = exp.Tty()
-			cmd.Stdout = exp.Tty()
-			cmd.Stderr = exp.Tty()
-
-			err = cmd.Start()
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if tfCmd.expectedCmdOutput != "" {
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-				if err != nil {
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-				}
-			}
-
-			lenInput := len(tfCmd.userInput)
-			lenInputOutput := len(tfCmd.postInputOutput)
-			if lenInput > 0 {
-				for i := 0; i < lenInput; i++ {
-					input := tfCmd.userInput[i]
-					exp.SendLine(input)
-					// use the index to find the corresponding
-					// output that matches the input.
-					if lenInputOutput-1 >= i {
-						output := tfCmd.postInputOutput[i]
-						_, err := exp.ExpectString(output)
-						if err != nil {
-							t.Fatal(err)
-						}
-					}
-				}
-			}
-
-			err = cmd.Wait()
-			if err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	if validations != nil {
-		validations(t, organization.Name)
-	}
-}
-
-func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_multi_workspace(t *testing.T) {
-	skipIfMissingEnvVar(t)
-	skipWithoutRemoteTerraformVersion(t)
-
-	ctx := context.Background()
-	operations := []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
 						_ = createWorkspace(t, orgName, tfe.WorkspaceCreateOptions{Name: tfe.String("app-one")})
@@ -987,8 +426,8 @@ func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_multi_workspace(t *
 						},
 					},
 				},
-	}
+			},
-	validations := func(t *testing.T, orgName string) {
+			validations: func(t *testing.T, orgName string) {
 				wsList, err := tfeClient.Workspaces.List(ctx, orgName, tfe.WorkspaceListOptions{
 					Tags: tfe.String("app"),
 				})
@@ -1012,72 +451,73 @@ func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_multi_workspace(t *
 				if len(ws.TagNames) == 0 {
 					t.Fatalf("expected workspaces 'app-two' to have tags.")
 				}
+			},
+		},
 	}
 
-	exp, err := expect.NewConsole(defaultOpts()...)
+	testRunner(t, cases, 1)
-	if err != nil {
+}
-		t.Fatal(err)
+
-	}
+func Test_migrate_remote_backend_multi_org(t *testing.T) {
-	defer exp.Close()
+	t.Parallel()
-
+	skipIfMissingEnvVar(t)
-	tmpDir, err := ioutil.TempDir("", "terraform-test")
+	skipWithoutRemoteTerraformVersion(t)
-	if err != nil {
+
-		t.Fatal(err)
+	ctx := context.Background()
-	}
+	cases := testCases{
-	defer os.RemoveAll(tmpDir)
+		"migrate remote backend name to tfc name": {
-
+			operations: []operationSets{
-	tf := e2e.NewBinary(terraformBin, tmpDir)
+				{
-	tf.AddEnv(cliConfigFileEnv)
+					prep: func(t *testing.T, orgName, dir string) {
-	defer tf.Close()
+						remoteWorkspace := "remote-workspace"
-
+						tfBlock := terraformConfigRemoteBackendName(orgName, remoteWorkspace)
-	organization, cleanup := createOrganization(t)
+						writeMainTF(t, tfBlock, dir)
-	defer cleanup()
+					},
-	for _, op := range operations {
+					commands: []tfCommand{
-		op.prep(t, organization.Name, tf.WorkDir())
+						{
-		for _, tfCmd := range op.commands {
+							command:           []string{"init"},
-			cmd := tf.Cmd(tfCmd.command...)
+							expectedCmdOutput: `Successfully configured the backend "remote"!`,
-			cmd.Stdin = exp.Tty()
+						},
-			cmd.Stdout = exp.Tty()
+						{
-			cmd.Stderr = exp.Tty()
+							command:         []string{"apply", "-auto-approve"},
-
+							postInputOutput: []string{`Apply complete!`},
-			err = cmd.Start()
+						},
-			if err != nil {
+					},
-				t.Fatal(err)
+				},
-			}
+				{
-
+					prep: func(t *testing.T, orgName, dir string) {
-			if tfCmd.expectedCmdOutput != "" {
+						wsName := "remote-workspace"
-				got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
+						tfBlock := terraformConfigCloudBackendName(orgName, wsName)
-				if err != nil {
+						writeMainTF(t, tfBlock, dir)
-					t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
+					},
-				}
+					commands: []tfCommand{
-			}
+						{
-
+							command:           []string{"init", "-ignore-remote-version"},
-			lenInput := len(tfCmd.userInput)
+							expectedCmdOutput: `Migrating from backend "remote" to Terraform Cloud.`,
-			lenInputOutput := len(tfCmd.postInputOutput)
+							userInput:         []string{"yes", "yes"},
-			if lenInput > 0 {
+							postInputOutput: []string{
-				for i := 0; i < lenInput; i++ {
+								`Should Terraform migrate your existing state?`,
-					input := tfCmd.userInput[i]
+								`Terraform Cloud has been successfully initialized!`},
-					exp.SendLine(input)
+						},
-					// use the index to find the corresponding
+						{
-					// output that matches the input.
+							command:           []string{"workspace", "show"},
-					if lenInputOutput-1 >= i {
+							expectedCmdOutput: `remote-workspace`,
-						output := tfCmd.postInputOutput[i]
+						},
-						_, err := exp.ExpectString(output)
+					},
-						if err != nil {
+				},
-							t.Fatal(err)
+			},
-						}
+			validations: func(t *testing.T, orgName string) {
-					}
+				expectedName := "remote-workspace"
-				}
+				ws, err := tfeClient.Workspaces.Read(ctx, orgName, expectedName)
-			}
+				if err != nil {
-
+					t.Fatal(err)
-			err = cmd.Wait()
+				}
-			if err != nil {
+				if ws == nil {
-				t.Fatal(err)
+					t.Fatalf("Expected workspace %s to be present, but is not.", expectedName)
-			}
+				}
-		}
+			},
-	}
+		},
-
+	}
-	if validations != nil {
+
-		validations(t, organization.Name)
+	testRunner(t, cases, 2)
-	}
 }

internal/cloud/e2e/migrate_state_single_to_tfc_test.go

@@ -2,25 +2,19 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 )
 
 func Test_migrate_single_to_tfc(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
 
-	cases := map[string]struct {
+	cases := testCases{
-		operations  []operationSets
-		validations func(t *testing.T, orgName string)
-	}{
 		"migrate using cloud workspace name strategy": {
 			operations: []operationSets{
 				{
@@ -128,76 +122,5 @@ func Test_migrate_single_to_tfc(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil {
-						t.Fatal(err)
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, organization.Name)
-			}
-		})
-	}
 }

internal/cloud/e2e/migrate_state_tfc_to_other_test.go

@@ -1,19 +1,14 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
-
-	expect "github.com/Netflix/go-expect"
-	"github.com/hashicorp/terraform/internal/e2e"
 )
 
 func Test_migrate_tfc_to_other(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
-	cases := map[string]struct {
+
-		operations []operationSets
+	cases := testCases{
-	}{
 		"migrate from cloud to local backend": {
 			operations: []operationSets{
 				{
@@ -46,71 +41,5 @@ func Test_migrate_tfc_to_other(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			// t.Parallel()
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-			}
-		})
-	}
 }

internal/cloud/e2e/migrate_state_tfc_to_tfc_test.go

@@ -2,31 +2,21 @@ package main
 
 import (
 	"context"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
 func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
+
 	ctx := context.Background()
 
-	cases := map[string]struct {
+	cases := testCases{
-		setup       func(t *testing.T) (string, func())
-		operations  []operationSets
-		validations func(t *testing.T, orgName string)
-	}{
 		"migrating from name to name": {
-			setup: func(t *testing.T) (string, func()) {
-				organization, cleanup := createOrganization(t)
-				return organization.Name, cleanup
-			},
 			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
@@ -91,10 +81,6 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
 			},
 		},
 		"migrating from name to tags": {
-			setup: func(t *testing.T) (string, func()) {
-				organization, cleanup := createOrganization(t)
-				return organization.Name, cleanup
-			},
 			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
@@ -153,10 +139,6 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
 			},
 		},
 		"migrating from name to tags without ignore-version flag": {
-			setup: func(t *testing.T) (string, func()) {
-				organization, cleanup := createOrganization(t)
-				return organization.Name, cleanup
-			},
 			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
@@ -218,94 +200,18 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		t.Run(name, func(t *testing.T) {
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			defer tf.Close()
-			tf.AddEnv(cliConfigFileEnv)
-
-			orgName, cleanup := tc.setup(t)
-			defer cleanup()
-			for _, op := range tc.operations {
-				op.prep(t, orgName, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err.Error())
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, orgName)
-			}
-		})
-	}
 }
 
 func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
 	ctx := context.Background()
 
-	cases := map[string]struct {
+	cases := testCases{
-		setup       func(t *testing.T) (string, func())
-		operations  []operationSets
-		validations func(t *testing.T, orgName string)
-	}{
 		"migrating from multiple workspaces via tags to name": {
-			setup: func(t *testing.T) (string, func()) {
-				organization, cleanup := createOrganization(t)
-				return organization.Name, cleanup
-			},
 			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
@@ -387,10 +293,6 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
 			},
 		},
 		"migrating from multiple workspaces via tags to other tags": {
-			setup: func(t *testing.T) (string, func()) {
-				organization, cleanup := createOrganization(t)
-				return organization.Name, cleanup
-			},
 			operations: []operationSets{
 				{
 					prep: func(t *testing.T, orgName, dir string) {
@@ -463,74 +365,5 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1)
-		t.Run(name, func(t *testing.T) {
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			defer tf.Close()
-			tf.AddEnv(cliConfigFileEnv)
-
-			orgName, cleanup := tc.setup(t)
-			defer cleanup()
-			for _, op := range tc.operations {
-				op.prep(t, orgName, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatal(err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil {
-						t.Fatal(err.Error())
-					}
-				}
-			}
-
-			if tc.validations != nil {
-				tc.validations(t, orgName)
-			}
-		})
-	}
 }

internal/cloud/e2e/run_variables_test.go

@@ -2,13 +2,9 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"testing"
 
-	expect "github.com/Netflix/go-expect"
 	tfe "github.com/hashicorp/go-tfe"
-	"github.com/hashicorp/terraform/internal/e2e"
 	tfversion "github.com/hashicorp/terraform/version"
 )
 
@@ -45,6 +41,7 @@ output "test_env" {
 }
 
 func Test_cloud_run_variables(t *testing.T) {
+	t.Parallel()
 	skipIfMissingEnvVar(t)
 	skipWithoutRemoteTerraformVersion(t)
 
@@ -80,76 +77,5 @@ func Test_cloud_run_variables(t *testing.T) {
 		},
 	}
 
-	for name, tc := range cases {
+	testRunner(t, cases, 1, "TF_CLI_ARGS=-no-color", "TF_VAR_baz=qux")
-		t.Run(name, func(t *testing.T) {
-			organization, cleanup := createOrganization(t)
-			defer cleanup()
-			exp, err := expect.NewConsole(defaultOpts()...)
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer exp.Close()
-
-			tmpDir, err := ioutil.TempDir("", "terraform-test")
-			if err != nil {
-				t.Fatal(err)
-			}
-			defer os.RemoveAll(tmpDir)
-
-			tf := e2e.NewBinary(terraformBin, tmpDir)
-			tf.AddEnv("TF_CLI_ARGS=-no-color")
-			tf.AddEnv("TF_VAR_baz=qux")
-			tf.AddEnv(cliConfigFileEnv)
-			defer tf.Close()
-
-			for _, op := range tc.operations {
-				op.prep(t, organization.Name, tf.WorkDir())
-				for _, tfCmd := range op.commands {
-					cmd := tf.Cmd(tfCmd.command...)
-					cmd.Stdin = exp.Tty()
-					cmd.Stdout = exp.Tty()
-					cmd.Stderr = exp.Tty()
-
-					err = cmd.Start()
-					if err != nil {
-						t.Fatal(err)
-					}
-
-					if tfCmd.expectedCmdOutput != "" {
-						got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
-						if err != nil {
-							t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
-						}
-					}
-
-					lenInput := len(tfCmd.userInput)
-					lenInputOutput := len(tfCmd.postInputOutput)
-					if lenInput > 0 {
-						for i := 0; i < lenInput; i++ {
-							input := tfCmd.userInput[i]
-							exp.SendLine(input)
-							// use the index to find the corresponding
-							// output that matches the input.
-							if lenInputOutput-1 >= i {
-								output := tfCmd.postInputOutput[i]
-								_, err := exp.ExpectString(output)
-								if err != nil {
-									t.Fatalf(`Expected command output "%s", but got %v `, tfCmd.expectedCmdOutput, err)
-								}
-							}
-						}
-					}
-
-					err = cmd.Wait()
-					if err != nil && !tfCmd.expectError {
-						t.Fatal(err)
-					}
-				}
-
-				if tc.validations != nil {
-					tc.validations(t, organization.Name)
-				}
-			}
-		})
-	}
 }

internal/command/arguments/show.go

@@ -0,0 +1,59 @@
+package arguments
+
+import (
+	"github.com/hashicorp/terraform/internal/tfdiags"
+)
+
+// Show represents the command-line arguments for the show command.
+type Show struct {
+	// Path is the path to the state file or plan file to be displayed. If
+	// unspecified, show will display the latest state snapshot.
+	Path string
+
+	// ViewType specifies which output format to use: human, JSON, or "raw".
+	ViewType ViewType
+}
+
+// ParseShow processes CLI arguments, returning a Show value and errors.
+// If errors are encountered, a Show value is still returned representing
+// the best effort interpretation of the arguments.
+func ParseShow(args []string) (*Show, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+	show := &Show{
+		Path: "",
+	}
+
+	var jsonOutput bool
+	cmdFlags := defaultFlagSet("show")
+	cmdFlags.BoolVar(&jsonOutput, "json", false, "json")
+
+	if err := cmdFlags.Parse(args); err != nil {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Failed to parse command-line flags",
+			err.Error(),
+		))
+	}
+
+	args = cmdFlags.Args()
+	if len(args) > 1 {
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Too many command line arguments",
+			"Expected at most one positional argument.",
+		))
+	}
+
+	if len(args) > 0 {
+		show.Path = args[0]
+	}
+
+	switch {
+	case jsonOutput:
+		show.ViewType = ViewJSON
+	default:
+		show.ViewType = ViewHuman
+	}
+
+	return show, diags
+}

internal/command/arguments/show_test.go

@@ -0,0 +1,99 @@
+package arguments
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/terraform/internal/tfdiags"
+)
+
+func TestParseShow_valid(t *testing.T) {
+	testCases := map[string]struct {
+		args []string
+		want *Show
+	}{
+		"defaults": {
+			nil,
+			&Show{
+				Path:     "",
+				ViewType: ViewHuman,
+			},
+		},
+		"json": {
+			[]string{"-json"},
+			&Show{
+				Path:     "",
+				ViewType: ViewJSON,
+			},
+		},
+		"path": {
+			[]string{"-json", "foo"},
+			&Show{
+				Path:     "foo",
+				ViewType: ViewJSON,
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got, diags := ParseShow(tc.args)
+			if len(diags) > 0 {
+				t.Fatalf("unexpected diags: %v", diags)
+			}
+			if *got != *tc.want {
+				t.Fatalf("unexpected result\n got: %#v\nwant: %#v", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestParseShow_invalid(t *testing.T) {
+	testCases := map[string]struct {
+		args      []string
+		want      *Show
+		wantDiags tfdiags.Diagnostics
+	}{
+		"unknown flag": {
+			[]string{"-boop"},
+			&Show{
+				Path:     "",
+				ViewType: ViewHuman,
+			},
+			tfdiags.Diagnostics{
+				tfdiags.Sourceless(
+					tfdiags.Error,
+					"Failed to parse command-line flags",
+					"flag provided but not defined: -boop",
+				),
+			},
+		},
+		"too many arguments": {
+			[]string{"-json", "bar", "baz"},
+			&Show{
+				Path:     "bar",
+				ViewType: ViewJSON,
+			},
+			tfdiags.Diagnostics{
+				tfdiags.Sourceless(
+					tfdiags.Error,
+					"Too many command line arguments",
+					"Expected at most one positional argument.",
+				),
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got, gotDiags := ParseShow(tc.args)
+			if *got != *tc.want {
+				t.Fatalf("unexpected result\n got: %#v\nwant: %#v", got, tc.want)
+			}
+			if !reflect.DeepEqual(gotDiags, tc.wantDiags) {
+				t.Errorf("wrong result\ngot: %s\nwant: %s", spew.Sdump(gotDiags), spew.Sdump(tc.wantDiags))
+			}
+		})
+	}
+}

internal/command/autocomplete.go

@@ -19,11 +19,7 @@ var completePredictModuleSource = complete.PredictAnything
 type completePredictSequence []complete.Predictor
 
 func (s completePredictSequence) Predict(a complete.Args) []string {
-	// Only one level of command is stripped off the prefix of a.Completed
+	// Nested subcommands do not require any placeholder entry for their subcommand name.
-	// here, so nested subcommands like "workspace new" will need to provide
-	// dummy entries (e.g. complete.PredictNothing) as placeholders for
-	// all but the first subcommand. For example, "workspace new" needs
-	// one placeholder for the argument "new".
 	idx := len(a.Completed)
 	if idx >= len(s) {
 		return nil

internal/command/e2etest/main_test.go

@@ -11,6 +11,18 @@ import (
 
 var terraformBin string
 
+// canRunGoBuild is a short-term compromise to account for the fact that we
+// have a small number of tests that work by building helper programs using
+// "go build" at runtime, but we can't do that in our isolated test mode
+// driven by the make-archive.sh script.
+//
+// FIXME: Rework this a bit so that we build the necessary helper programs
+// (test plugins, etc) as part of the initial suite setup, and in the
+// make-archive.sh script, so that we can run all of the tests in both
+// situations with the tests just using the executable already built for
+// them, as we do for terraformBin.
+var canRunGoBuild bool
+
 func TestMain(m *testing.M) {
 	teardown := setup()
 	code := m.Run()
@@ -21,10 +33,10 @@ func TestMain(m *testing.M) {
 func setup() func() {
 	if terraformBin != "" {
 		// this is pre-set when we're running in a binary produced from
-		// the make-archive.sh script, since that builds a ready-to-go
+		// the make-archive.sh script, since that is for testing an
-		// binary into the archive. However, we do need to turn it into
+		// executable obtained from a real release package. However, we do
-		// an absolute path so that we can find it when we change the
+		// need to turn it into an absolute path so that we can find it
-		// working directory during tests.
+		// when we change the working directory during tests.
 		var err error
 		terraformBin, err = filepath.Abs(terraformBin)
 		if err != nil {
@@ -38,6 +50,11 @@ func setup() func() {
 	// Make the executable available for use in tests
 	terraformBin = tmpFilename
 
+	// Tests running in the ad-hoc testing mode are allowed to use "go build"
+	// and similar to produce other test executables.
+	// (See the comment on this variable's declaration for more information.)
+	canRunGoBuild = true
+
 	return func() {
 		os.Remove(tmpFilename)
 	}

internal/command/e2etest/make-archive.sh

@@ -13,9 +13,12 @@
 # and then executed as follows:
 #    set TF_ACC=1
 #    ./e2etest.exe
-# Since the test archive includes both the test fixtures and the compiled
+#
-# terraform executable along with this test program, the result is
+# Because separated e2etest harnesses are intended for testing against "real"
-# self-contained and does not require a local Go compiler on the target system.
+# release executables, the generated archives don't include a copy of
+# the Terraform executable. Instead, the caller of the tests must retrieve
+# and extract a release package into the working directory before running
+# the e2etest executable, so that "e2etest" can find and execute it.
 
 set +euo pipefail
 
@@ -33,10 +36,6 @@ mkdir -p "$OUTDIR"
 # We need the test fixtures available when we run the tests.
 cp -r testdata "$OUTDIR/testdata"
 
-# Bundle a copy of our binary so the target system doesn't need the go
-# compiler installed.
-go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform
-
 # Build the test program
 go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/internal/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/internal/command/e2etest
 

internal/command/e2etest/primary_test.go

@@ -204,13 +204,13 @@ func TestPrimaryChdirOption(t *testing.T) {
 	}
 
 	gotOutput := state.RootModule().OutputValues["cwd"]
-	wantOutputValue := cty.StringVal(tf.Path()) // path.cwd returns the original path, because path.root is how we get the overridden path
+	wantOutputValue := cty.StringVal(filepath.ToSlash(tf.Path())) // path.cwd returns the original path, because path.root is how we get the overridden path
 	if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
 		t.Errorf("incorrect value for cwd output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
 	}
 
 	gotOutput = state.RootModule().OutputValues["root"]
-	wantOutputValue = cty.StringVal(tf.Path("subdir")) // path.root is a relative path, but the text fixture uses abspath on it.
+	wantOutputValue = cty.StringVal(filepath.ToSlash(tf.Path("subdir"))) // path.root is a relative path, but the text fixture uses abspath on it.
 	if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
 		t.Errorf("incorrect value for root output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
 	}

internal/command/e2etest/provider_dev_test.go

@@ -18,6 +18,14 @@ import (
 // we normally do, so they can just overwrite the same local executable
 // in-place to iterate faster.
 func TestProviderDevOverrides(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
 	t.Parallel()
 
 	tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override")

internal/command/e2etest/provider_plugin_test.go

@@ -13,6 +13,14 @@ import (
 // TestProviderProtocols verifies that Terraform can execute provider plugins
 // with both supported protocol versions.
 func TestProviderProtocols(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provider executable")
+	}
 	t.Parallel()
 
 	tf := e2e.NewBinary(terraformBin, "testdata/provider-plugin")

internal/command/e2etest/providers_tamper_test.go

@@ -41,12 +41,16 @@ func TestProviderTampering(t *testing.T) {
 
 	seedDir := tf.WorkDir()
 	const providerVersion = "3.1.0" // must match the version in the fixture config
-	pluginDir := ".terraform/providers/registry.terraform.io/hashicorp/null/" + providerVersion + "/" + getproviders.CurrentPlatform.String()
+	pluginDir := filepath.Join(".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String())
-	pluginExe := pluginDir + "/terraform-provider-null_v" + providerVersion + "_x5"
+	pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5")
 	if getproviders.CurrentPlatform.OS == "windows" {
 		pluginExe += ".exe" // ugh
 	}
 
+	// filepath.Join here to make sure we get the right path separator
+	// for whatever OS we're running these tests on.
+	providerCacheDir := filepath.Join(".terraform", "providers")
+
 	t.Run("cache dir totally gone", func(t *testing.T) {
 		tf := e2e.NewBinary(terraformBin, seedDir)
 		defer tf.Close()
@@ -61,7 +65,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 		if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -128,7 +132,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 		if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -237,7 +241,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 	})
@@ -260,7 +264,7 @@ func TestProviderTampering(t *testing.T) {
 		if err == nil {
 			t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
 		}
-		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
+		if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
 			t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
 		}
 	})

internal/command/e2etest/provisioner_plugin_test.go

@@ -12,6 +12,14 @@ import (
 // TestProvisionerPlugin is a test that terraform can execute a 3rd party
 // provisioner plugin.
 func TestProvisionerPlugin(t *testing.T) {
+	if !canRunGoBuild {
+		// We're running in a separate-build-then-run context, so we can't
+		// currently execute this test which depends on being able to build
+		// new executable at runtime.
+		//
+		// (See the comment on canRunGoBuild's declaration for more information.)
+		t.Skip("can't run without building a new provisioner executable")
+	}
 	t.Parallel()
 
 	// This test reaches out to releases.hashicorp.com to download the

internal/command/jsonplan/plan.go

@@ -118,7 +118,7 @@ func Marshal(
 	output := newPlan()
 	output.TerraformVersion = version.String()
 
-	err := output.marshalPlanVariables(p.VariableValues, schemas)
+	err := output.marshalPlanVariables(p.VariableValues, config.Module.Variables)
 	if err != nil {
 		return nil, fmt.Errorf("error in marshalPlanVariables: %s", err)
 	}
@@ -183,11 +183,7 @@ func Marshal(
 	return ret, err
 }
 
-func (p *plan) marshalPlanVariables(vars map[string]plans.DynamicValue, schemas *terraform.Schemas) error {
+func (p *plan) marshalPlanVariables(vars map[string]plans.DynamicValue, decls map[string]*configs.Variable) error {
-	if len(vars) == 0 {
-		return nil
-	}
-
 	p.Variables = make(variables, len(vars))
 
 	for k, v := range vars {
@@ -203,6 +199,41 @@ func (p *plan) marshalPlanVariables(vars map[string]plans.DynamicValue, schemas
 			Value: valJSON,
 		}
 	}
+
+	// In Terraform v1.1 and earlier we had some confusion about which subsystem
+	// of Terraform was the one responsible for substituting in default values
+	// for unset module variables, with root module variables being handled in
+	// three different places while child module variables were only handled
+	// during the Terraform Core graph walk.
+	//
+	// For Terraform v1.2 and later we rationalized that by having the Terraform
+	// Core graph walk always be responsible for selecting defaults regardless
+	// of root vs. child module, but unfortunately our earlier accidental
+	// misbehavior bled out into the public interface by making the defaults
+	// show up in the "vars" map to this function. Those are now correctly
+	// omitted (so that the plan file only records the variables _actually_
+	// set by the caller) but consumers of the JSON plan format may be depending
+	// on our old behavior and so we'll fake it here just in time so that
+	// outside consumers won't see a behavior change.
+	for name, decl := range decls {
+		if _, ok := p.Variables[name]; ok {
+			continue
+		}
+		if val := decl.Default; val != cty.NilVal {
+			valJSON, err := ctyjson.Marshal(val, val.Type())
+			if err != nil {
+				return err
+			}
+			p.Variables[name] = &variable{
+				Value: valJSON,
+			}
+		}
+	}
+
+	if len(p.Variables) == 0 {
+		p.Variables = nil // omit this property if there are no variables to describe
+	}
+
 	return nil
 }
 

internal/command/show.go

@@ -7,14 +7,13 @@ import (
 
 	"github.com/hashicorp/terraform/internal/backend"
 	"github.com/hashicorp/terraform/internal/command/arguments"
-	"github.com/hashicorp/terraform/internal/command/format"
-	"github.com/hashicorp/terraform/internal/command/jsonplan"
-	"github.com/hashicorp/terraform/internal/command/jsonstate"
 	"github.com/hashicorp/terraform/internal/command/views"
+	"github.com/hashicorp/terraform/internal/configs"
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/hashicorp/terraform/internal/plans/planfile"
 	"github.com/hashicorp/terraform/internal/states/statefile"
 	"github.com/hashicorp/terraform/internal/states/statemgr"
+	"github.com/hashicorp/terraform/internal/terraform"
 	"github.com/hashicorp/terraform/internal/tfdiags"
 )
 
@@ -24,173 +23,40 @@ type ShowCommand struct {
 	Meta
 }
 
-func (c *ShowCommand) Run(args []string) int {
+func (c *ShowCommand) Run(rawArgs []string) int {
-	args = c.Meta.process(args)
+	// Parse and apply global view arguments
-	cmdFlags := c.Meta.defaultFlagSet("show")
+	common, rawArgs := arguments.ParseView(rawArgs)
-	var jsonOutput bool
+	c.View.Configure(common)
-	cmdFlags.BoolVar(&jsonOutput, "json", false, "produce JSON output")
+
-	cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
+	// Parse and validate flags
-	if err := cmdFlags.Parse(args); err != nil {
+	args, diags := arguments.ParseShow(rawArgs)
-		c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error()))
+	if diags.HasErrors() {
+		c.View.Diagnostics(diags)
+		c.View.HelpPrompt("show")
 		return 1
 	}
 
-	args = cmdFlags.Args()
+	// Set up view
-	if len(args) > 2 {
+	view := views.NewShow(args.ViewType, c.View)
-		c.Ui.Error(
-			"The show command expects at most two arguments.\n The path to a " +
-				"Terraform state or plan file, and optionally -json for json output.\n")
-		cmdFlags.Usage()
-		return 1
-	}
 
 	// Check for user-supplied plugin path
 	var err error
 	if c.pluginPath, err = c.loadPluginPath(); err != nil {
-		c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
+		diags = diags.Append(fmt.Errorf("error loading plugin path: %s", err))
+		view.Diagnostics(diags)
 		return 1
 	}
 
-	var diags tfdiags.Diagnostics
+	// Get the data we need to display
-
+	plan, stateFile, config, schemas, showDiags := c.show(args.Path)
-	// Load the backend
+	diags = diags.Append(showDiags)
-	b, backendDiags := c.Backend(nil)
+	if showDiags.HasErrors() {
-	diags = diags.Append(backendDiags)
+		view.Diagnostics(diags)
-	if backendDiags.HasErrors() {
-		c.showDiagnostics(diags)
 		return 1
 	}
 
-	// We require a local backend
+	// Display the data
-	local, ok := b.(backend.Local)
+	return view.Display(config, plan, stateFile, schemas)
-	if !ok {
-		c.showDiagnostics(diags) // in case of any warnings in here
-		c.Ui.Error(ErrUnsupportedLocalOp)
-		return 1
-	}
-
-	// This is a read-only command
-	c.ignoreRemoteVersionConflict(b)
-
-	// the show command expects the config dir to always be the cwd
-	cwd, err := os.Getwd()
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("Error getting cwd: %s", err))
-		return 1
-	}
-
-	// Determine if a planfile was passed to the command
-	var planFile *planfile.Reader
-	if len(args) > 0 {
-		// We will handle error checking later on - this is just required to
-		// load the local context if the given path is successfully read as
-		// a planfile.
-		planFile, _ = c.PlanFile(args[0])
-	}
-
-	// Build the operation
-	opReq := c.Operation(b)
-	opReq.ConfigDir = cwd
-	opReq.PlanFile = planFile
-	opReq.ConfigLoader, err = c.initConfigLoader()
-	opReq.AllowUnsetVariables = true
-	if err != nil {
-		diags = diags.Append(err)
-		c.showDiagnostics(diags)
-		return 1
-	}
-
-	// Get the context
-	lr, _, ctxDiags := local.LocalRun(opReq)
-	diags = diags.Append(ctxDiags)
-	if ctxDiags.HasErrors() {
-		c.showDiagnostics(diags)
-		return 1
-	}
-
-	// Get the schemas from the context
-	schemas, moreDiags := lr.Core.Schemas(lr.Config, lr.InputState)
-	diags = diags.Append(moreDiags)
-	if moreDiags.HasErrors() {
-		c.showDiagnostics(diags)
-		return 1
-	}
-
-	var planErr, stateErr error
-	var plan *plans.Plan
-	var stateFile *statefile.File
-
-	// if a path was provided, try to read it as a path to a planfile
-	// if that fails, try to read the cli argument as a path to a statefile
-	if len(args) > 0 {
-		path := args[0]
-		plan, stateFile, planErr = getPlanFromPath(path)
-		if planErr != nil {
-			stateFile, stateErr = getStateFromPath(path)
-			if stateErr != nil {
-				c.Ui.Error(fmt.Sprintf(
-					"Terraform couldn't read the given file as a state or plan file.\n"+
-						"The errors while attempting to read the file as each format are\n"+
-						"shown below.\n\n"+
-						"State read error: %s\n\nPlan read error: %s",
-					stateErr,
-					planErr))
-				return 1
-			}
-		}
-	} else {
-		env, err := c.Workspace()
-		if err != nil {
-			c.Ui.Error(fmt.Sprintf("Error selecting workspace: %s", err))
-			return 1
-		}
-		stateFile, stateErr = getStateFromEnv(b, env)
-		if stateErr != nil {
-			c.Ui.Error(stateErr.Error())
-			return 1
-		}
-	}
-
-	if plan != nil {
-		if jsonOutput {
-			config := lr.Config
-			jsonPlan, err := jsonplan.Marshal(config, plan, stateFile, schemas)
-
-			if err != nil {
-				c.Ui.Error(fmt.Sprintf("Failed to marshal plan to json: %s", err))
-				return 1
-			}
-			c.Ui.Output(string(jsonPlan))
-			return 0
-		}
-
-		view := views.NewShow(arguments.ViewHuman, c.View)
-		view.Plan(plan, schemas)
-		return 0
-	}
-
-	if jsonOutput {
-		// At this point, it is possible that there is neither state nor a plan.
-		// That's ok, we'll just return an empty object.
-		jsonState, err := jsonstate.Marshal(stateFile, schemas)
-		if err != nil {
-			c.Ui.Error(fmt.Sprintf("Failed to marshal state to json: %s", err))
-			return 1
-		}
-		c.Ui.Output(string(jsonState))
-	} else {
-		if stateFile == nil {
-			c.Ui.Output("No state.")
-			return 0
-		}
-		c.Ui.Output(format.State(&format.StateOpts{
-			State:   stateFile.State,
-			Color:   c.Colorize(),
-			Schemas: schemas,
-		}))
-	}
-
-	return 0
 }
 
 func (c *ShowCommand) Help() string {
@@ -214,52 +80,171 @@ func (c *ShowCommand) Synopsis() string {
 	return "Show the current state or a saved plan"
 }
 
-// getPlanFromPath returns a plan and statefile if the user-supplied path points
+func (c *ShowCommand) show(path string) (*plans.Plan, *statefile.File, *configs.Config, *terraform.Schemas, tfdiags.Diagnostics) {
-// to a planfile. If both plan and error are nil, the path is likely a
+	var diags, showDiags tfdiags.Diagnostics
-// directory. An error could suggest that the given path points to a statefile.
+	var plan *plans.Plan
-func getPlanFromPath(path string) (*plans.Plan, *statefile.File, error) {
+	var stateFile *statefile.File
-	pr, err := planfile.Open(path)
+	var config *configs.Config
-	if err != nil {
+	var schemas *terraform.Schemas
-		return nil, nil, err
+
+	// No plan file or state file argument provided,
+	// so get the latest state snapshot
+	if path == "" {
+		stateFile, showDiags = c.showFromLatestStateSnapshot()
+		diags = diags.Append(showDiags)
+		if showDiags.HasErrors() {
+			return plan, stateFile, config, schemas, diags
 		}
-	plan, err := pr.ReadPlan()
-	if err != nil {
-		return nil, nil, err
 	}
 
-	stateFile, err := pr.ReadStateFile()
+	// Plan file or state file argument provided,
-	return plan, stateFile, err
+	// so try to load the argument as a plan file first.
+	// If that fails, try to load it as a statefile.
+	if path != "" {
+		plan, stateFile, config, showDiags = c.showFromPath(path)
+		diags = diags.Append(showDiags)
+		if showDiags.HasErrors() {
+			return plan, stateFile, config, schemas, diags
+		}
+	}
+
+	// Get schemas, if possible
+	if config != nil || stateFile != nil {
+		opts, err := c.contextOpts()
+		if err != nil {
+			diags = diags.Append(err)
+			return plan, stateFile, config, schemas, diags
+		}
+		tfCtx, ctxDiags := terraform.NewContext(opts)
+		diags = diags.Append(ctxDiags)
+		if ctxDiags.HasErrors() {
+			return plan, stateFile, config, schemas, diags
+		}
+		var schemaDiags tfdiags.Diagnostics
+		schemas, schemaDiags = tfCtx.Schemas(config, stateFile.State)
+		diags = diags.Append(schemaDiags)
+		if schemaDiags.HasErrors() {
+			return plan, stateFile, config, schemas, diags
+		}
+	}
+
+	return plan, stateFile, config, schemas, diags
+}
+func (c *ShowCommand) showFromLatestStateSnapshot() (*statefile.File, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	// Load the backend
+	b, backendDiags := c.Backend(nil)
+	diags = diags.Append(backendDiags)
+	if backendDiags.HasErrors() {
+		return nil, diags
+	}
+	c.ignoreRemoteVersionConflict(b)
+
+	// Load the workspace
+	workspace, err := c.Workspace()
+	if err != nil {
+		diags = diags.Append(fmt.Errorf("error selecting workspace: %s", err))
+		return nil, diags
+	}
+
+	// Get the latest state snapshot from the backend for the current workspace
+	stateFile, stateErr := getStateFromBackend(b, workspace)
+	if stateErr != nil {
+		diags = diags.Append(stateErr.Error())
+		return nil, diags
+	}
+
+	return stateFile, diags
+}
+
+func (c *ShowCommand) showFromPath(path string) (*plans.Plan, *statefile.File, *configs.Config, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+	var planErr, stateErr error
+	var plan *plans.Plan
+	var stateFile *statefile.File
+	var config *configs.Config
+
+	// Try to get the plan file and associated data from
+	// the path argument. If that fails, try to get the
+	// statefile from the path argument.
+	plan, stateFile, config, planErr = getPlanFromPath(path)
+	if planErr != nil {
+		stateFile, stateErr = getStateFromPath(path)
+		if stateErr != nil {
+			diags = diags.Append(
+				tfdiags.Sourceless(
+					tfdiags.Error,
+					"Failed to read the given file as a state or plan file",
+					fmt.Sprintf("State read error: %s\n\nPlan read error: %s", stateErr, planErr),
+				),
+			)
+			return nil, nil, nil, diags
+		}
+	}
+	return plan, stateFile, config, diags
+}
+
+// getPlanFromPath returns a plan, statefile, and config if the user-supplied
+// path points to a plan file. If both plan and error are nil, the path is likely
+// a directory. An error could suggest that the given path points to a statefile.
+func getPlanFromPath(path string) (*plans.Plan, *statefile.File, *configs.Config, error) {
+	planReader, err := planfile.Open(path)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	// Get plan
+	plan, err := planReader.ReadPlan()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	// Get statefile
+	stateFile, err := planReader.ReadStateFile()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	// Get config
+	config, diags := planReader.ReadConfig()
+	if diags.HasErrors() {
+		return nil, nil, nil, diags.Err()
+	}
+
+	return plan, stateFile, config, err
 }
 
 // getStateFromPath returns a statefile if the user-supplied path points to a statefile.
 func getStateFromPath(path string) (*statefile.File, error) {
-	f, err := os.Open(path)
+	file, err := os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("Error loading statefile: %s", err)
 	}
-	defer f.Close()
+	defer file.Close()
 
 	var stateFile *statefile.File
-	stateFile, err = statefile.Read(f)
+	stateFile, err = statefile.Read(file)
 	if err != nil {
 		return nil, fmt.Errorf("Error reading %s as a statefile: %s", path, err)
 	}
 	return stateFile, nil
 }
 
-// getStateFromEnv returns the State for the current workspace, if available.
+// getStateFromBackend returns the State for the current workspace, if available.
-func getStateFromEnv(b backend.Backend, env string) (*statefile.File, error) {
+func getStateFromBackend(b backend.Backend, workspace string) (*statefile.File, error) {
-	// Get the state
+	// Get the state store for the given workspace
-	stateStore, err := b.StateMgr(env)
+	stateStore, err := b.StateMgr(workspace)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to load state manager: %s", err)
 	}
 
+	// Refresh the state store with the latest state snapshot from persistent storage
 	if err := stateStore.RefreshState(); err != nil {
 		return nil, fmt.Errorf("Failed to load state: %s", err)
 	}
 
-	sf := statemgr.Export(stateStore)
+	// Get the latest state snapshot and return it
-
+	stateFile := statemgr.Export(stateStore)
-	return sf, nil
+	return stateFile, nil
 }

internal/command/show_test.go

@@ -2,7 +2,6 @@ package command
 
 import (
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -15,18 +14,18 @@ import (
 	"github.com/hashicorp/terraform/internal/plans"
 	"github.com/hashicorp/terraform/internal/providers"
 	"github.com/hashicorp/terraform/internal/states"
+	"github.com/hashicorp/terraform/internal/states/statemgr"
 	"github.com/hashicorp/terraform/internal/terraform"
+	"github.com/hashicorp/terraform/version"
 	"github.com/mitchellh/cli"
 	"github.com/zclconf/go-cty/cty"
 )
 
-func TestShow(t *testing.T) {
+func TestShow_badArgs(t *testing.T) {
-	ui := new(cli.MockUi)
+	view, done := testView(t)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
@@ -34,40 +33,99 @@ func TestShow(t *testing.T) {
 	args := []string{
 		"bad",
 		"bad",
+		"-no-color",
 	}
-	if code := c.Run(args); code != 1 {
+
-		t.Fatalf("bad: \n%s", ui.OutputWriter.String())
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 1 {
+		t.Fatalf("unexpected exit status %d; want 1\ngot: %s", code, output.Stdout())
 	}
 }
 
-func TestShow_noArgs(t *testing.T) {
+func TestShow_noArgsNoState(t *testing.T) {
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	code := c.Run([]string{})
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
+	}
+
+	got := output.Stdout()
+	want := `No state.`
+	if !strings.Contains(got, want) {
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
+	}
+}
+
+func TestShow_noArgsWithState(t *testing.T) {
 	// Get a temp cwd
 	tmp, cwd := testCwd(t)
 	defer testFixCwd(t, tmp, cwd)
 	// Create the default state
 	testStateFileDefault(t, testState())
 
-	ui := new(cli.MockUi)
+	view, done := testView(t)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
-	if code := c.Run([]string{}); code != 0 {
+	code := c.Run([]string{})
-		t.Fatalf("bad: \n%s", ui.OutputWriter.String())
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 
-	if !strings.Contains(ui.OutputWriter.String(), "# test_instance.foo:") {
+	got := output.Stdout()
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	want := `# test_instance.foo:`
+	if !strings.Contains(got, want) {
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
+	}
+}
+
+func TestShow_argsWithState(t *testing.T) {
+	// Create the default state
+	statePath := testStateFile(t, testState())
+	stateDir := filepath.Dir(statePath)
+	defer os.RemoveAll(stateDir)
+	defer testChdir(t, stateDir)()
+
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	path := filepath.Base(statePath)
+	args := []string{
+		path,
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 }
 
 // https://github.com/hashicorp/terraform/issues/21462
-func TestShow_aliasedProvider(t *testing.T) {
+func TestShow_argsWithStateAliasedProvider(t *testing.T) {
 	// Create the default state with aliased resource
 	testState := states.BuildState(func(s *states.SyncState) {
 		s.SetResourceInstanceCurrent(
@@ -93,103 +151,198 @@ func TestShow_aliasedProvider(t *testing.T) {
 	defer os.RemoveAll(stateDir)
 	defer testChdir(t, stateDir)()
 
-	ui := new(cli.MockUi)
+	view, done := testView(t)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
-	// the statefile created by testStateFile is named state.tfstate
+	path := filepath.Base(statePath)
-	args := []string{"state.tfstate"}
+	args := []string{
-	if code := c.Run(args); code != 0 {
+		path,
-		t.Fatalf("bad exit code: \n%s", ui.OutputWriter.String())
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 
-	if strings.Contains(ui.OutputWriter.String(), "# missing schema for provider \"test.alias\"") {
+	got := output.Stdout()
-		t.Fatalf("bad output: \n%s", ui.OutputWriter.String())
+	want := `# missing schema for provider \"test.alias\"`
+	if strings.Contains(got, want) {
+		t.Fatalf("unexpected output\ngot: %s", got)
 	}
 }
 
-func TestShow_noArgsNoState(t *testing.T) {
+func TestShow_argsPlanFileDoesNotExist(t *testing.T) {
-	// Create the default state
+	view, done := testView(t)
-	statePath := testStateFile(t, testState())
-	stateDir := filepath.Dir(statePath)
-	defer os.RemoveAll(stateDir)
-	defer testChdir(t, stateDir)()
-
-	ui := new(cli.MockUi)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
-	// the statefile created by testStateFile is named state.tfstate
+	args := []string{
-	args := []string{"state.tfstate"}
+		"doesNotExist.tfplan",
-	if code := c.Run(args); code != 0 {
+		"-no-color",
-		t.Fatalf("bad: \n%s", ui.OutputWriter.String())
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 1 {
+		t.Fatalf("unexpected exit status %d; want 1\ngot: %s", code, output.Stdout())
+	}
+
+	got := output.Stderr()
+	want := `Plan read error: open doesNotExist.tfplan:`
+	if !strings.Contains(got, want) {
+		t.Errorf("unexpected output\ngot: %s\nwant:\n%s", got, want)
+	}
+}
+
+func TestShow_argsStatefileDoesNotExist(t *testing.T) {
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	args := []string{
+		"doesNotExist.tfstate",
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 1 {
+		t.Fatalf("unexpected exit status %d; want 1\ngot: %s", code, output.Stdout())
+	}
+
+	got := output.Stderr()
+	want := `State read error: Error loading statefile:`
+	if !strings.Contains(got, want) {
+		t.Errorf("unexpected output\ngot: %s\nwant:\n%s", got, want)
+	}
+}
+
+func TestShow_json_argsPlanFileDoesNotExist(t *testing.T) {
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	args := []string{
+		"-json",
+		"doesNotExist.tfplan",
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 1 {
+		t.Fatalf("unexpected exit status %d; want 1\ngot: %s", code, output.Stdout())
+	}
+
+	got := output.Stderr()
+	want := `Plan read error: open doesNotExist.tfplan:`
+	if !strings.Contains(got, want) {
+		t.Errorf("unexpected output\ngot: %s\nwant:\n%s", got, want)
+	}
+}
+
+func TestShow_json_argsStatefileDoesNotExist(t *testing.T) {
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	args := []string{
+		"-json",
+		"doesNotExist.tfstate",
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 1 {
+		t.Fatalf("unexpected exit status %d; want 1\ngot: %s", code, output.Stdout())
+	}
+
+	got := output.Stderr()
+	want := `State read error: Error loading statefile:`
+	if !strings.Contains(got, want) {
+		t.Errorf("unexpected output\ngot: %s\nwant:\n%s", got, want)
 	}
 }
 
 func TestShow_planNoop(t *testing.T) {
 	planPath := testPlanFileNoop(t)
 
-	ui := cli.NewMockUi()
 	view, done := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
 	args := []string{
 		planPath,
+		"-no-color",
 	}
-	if code := c.Run(args); code != 0 {
+	code := c.Run(args)
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 
+	got := output.Stdout()
 	want := `No changes. Your infrastructure matches the configuration.`
-	got := done(t).Stdout()
 	if !strings.Contains(got, want) {
-		t.Errorf("missing expected output\nwant: %s\ngot:\n%s", want, got)
+		t.Errorf("unexpected output\ngot: %s\nwant:\n%s", got, want)
 	}
 }
 
 func TestShow_planWithChanges(t *testing.T) {
 	planPathWithChanges := showFixturePlanFile(t, plans.DeleteThenCreate)
 
-	ui := cli.NewMockUi()
 	view, done := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(showFixtureProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
 	args := []string{
 		planPathWithChanges,
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 
-	if code := c.Run(args); code != 0 {
+	got := output.Stdout()
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
-	}
-
 	want := `test_instance.foo must be replaced`
-	got := done(t).Stdout()
 	if !strings.Contains(got, want) {
-		t.Errorf("missing expected output\nwant: %s\ngot:\n%s", want, got)
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
 	}
 }
 
@@ -237,30 +390,34 @@ func TestShow_planWithForceReplaceChange(t *testing.T) {
 		plan,
 	)
 
-	ui := cli.NewMockUi()
 	view, done := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(showFixtureProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
 	args := []string{
 		planFilePath,
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 
-	if code := c.Run(args); code != 0 {
+	got := output.Stdout()
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	want := `test_instance.foo will be replaced, as requested`
+	if !strings.Contains(got, want) {
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
 	}
 
-	got := done(t).Stdout()
+	want = `Plan: 1 to add, 0 to change, 1 to destroy.`
-	if want := `test_instance.foo will be replaced, as requested`; !strings.Contains(got, want) {
+	if !strings.Contains(got, want) {
-		t.Errorf("wrong output\ngot:\n%s\n\nwant substring: %s", got, want)
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
-	}
-	if want := `Plan: 1 to add, 0 to change, 1 to destroy.`; !strings.Contains(got, want) {
-		t.Errorf("wrong output\ngot:\n%s\n\nwant substring: %s", got, want)
 	}
 
 }
@@ -268,12 +425,10 @@ func TestShow_planWithForceReplaceChange(t *testing.T) {
 func TestShow_plan_json(t *testing.T) {
 	planPath := showFixturePlanFile(t, plans.Create)
 
-	ui := new(cli.MockUi)
+	view, done := testView(t)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(showFixtureProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
@@ -281,9 +436,13 @@ func TestShow_plan_json(t *testing.T) {
 	args := []string{
 		"-json",
 		planPath,
+		"-no-color",
 	}
-	if code := c.Run(args); code != 0 {
+	code := c.Run(args)
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 }
 
@@ -292,21 +451,23 @@ func TestShow_state(t *testing.T) {
 	statePath := testStateFile(t, originalState)
 	defer os.RemoveAll(filepath.Dir(statePath))
 
-	ui := new(cli.MockUi)
+	view, done := testView(t)
-	view, _ := testView(t)
 	c := &ShowCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
-			Ui:               ui,
 			View:             view,
 		},
 	}
 
 	args := []string{
 		statePath,
+		"-no-color",
 	}
-	if code := c.Run(args); code != 0 {
+	code := c.Run(args)
-		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
 	}
 }
 
@@ -337,18 +498,15 @@ func TestShow_json_output(t *testing.T) {
 			defer close()
 
 			p := showFixtureProvider()
-			ui := new(cli.MockUi)
-			view, _ := testView(t)
-			m := Meta{
-				testingOverrides: metaOverridesForProvider(p),
-				Ui:               ui,
-				View:             view,
-				ProviderSource:   providerSource,
-			}
 
 			// init
+			ui := new(cli.MockUi)
 			ic := &InitCommand{
-				Meta: m,
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					Ui:               ui,
+					ProviderSource:   providerSource,
+				},
 			}
 			if code := ic.Run([]string{}); code != 0 {
 				if expectError {
@@ -358,22 +516,35 @@ func TestShow_json_output(t *testing.T) {
 				t.Fatalf("init failed\n%s", ui.ErrorWriter)
 			}
 
+			// plan
+			planView, planDone := testView(t)
 			pc := &PlanCommand{
-				Meta: m,
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					View:             planView,
+					ProviderSource:   providerSource,
+				},
 			}
 
 			args := []string{
 				"-out=terraform.plan",
 			}
 
-			if code := pc.Run(args); code != 0 {
+			code := pc.Run(args)
-				t.Fatalf("wrong exit status %d; want 0\nstderr: %s", code, ui.ErrorWriter.String())
+			planOutput := planDone(t)
+
+			if code != 0 {
+				t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, planOutput.Stderr())
 			}
 
-			// flush the plan output from the mock ui
+			// show
-			ui.OutputWriter.Reset()
+			showView, showDone := testView(t)
 			sc := &ShowCommand{
-				Meta: m,
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					View:             showView,
+					ProviderSource:   providerSource,
+				},
 			}
 
 			args = []string{
@@ -381,25 +552,27 @@ func TestShow_json_output(t *testing.T) {
 				"terraform.plan",
 			}
 			defer os.Remove("terraform.plan")
+			code = sc.Run(args)
+			showOutput := showDone(t)
 
-			if code := sc.Run(args); code != 0 {
+			if code != 0 {
-				t.Fatalf("wrong exit status %d; want 0\nstderr: %s", code, ui.ErrorWriter.String())
+				t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, showOutput.Stderr())
 			}
 
-			// compare ui output to wanted output
+			// compare view output to wanted output
 			var got, want plan
 
-			gotString := ui.OutputWriter.String()
+			gotString := showOutput.Stdout()
 			json.Unmarshal([]byte(gotString), &got)
 
 			wantFile, err := os.Open("output.json")
 			if err != nil {
-				t.Fatalf("err: %s", err)
+				t.Fatalf("unexpected err: %s", err)
 			}
 			defer wantFile.Close()
 			byteValue, err := ioutil.ReadAll(wantFile)
 			if err != nil {
-				t.Fatalf("err: %s", err)
+				t.Fatalf("unexpected err: %s", err)
 			}
 			json.Unmarshal([]byte(byteValue), &want)
 
@@ -421,43 +594,48 @@ func TestShow_json_output_sensitive(t *testing.T) {
 	defer close()
 
 	p := showFixtureSensitiveProvider()
-	ui := new(cli.MockUi)
-	view, _ := testView(t)
-	m := Meta{
-		testingOverrides: metaOverridesForProvider(p),
-		Ui:               ui,
-		View:             view,
-		ProviderSource:   providerSource,
-	}
 
 	// init
+	ui := new(cli.MockUi)
 	ic := &InitCommand{
-		Meta: m,
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(p),
+			Ui:               ui,
+			ProviderSource:   providerSource,
+		},
 	}
 	if code := ic.Run([]string{}); code != 0 {
 		t.Fatalf("init failed\n%s", ui.ErrorWriter)
 	}
 
-	// flush init output
+	// plan
-	ui.OutputWriter.Reset()
+	planView, planDone := testView(t)
-
 	pc := &PlanCommand{
-		Meta: m,
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(p),
+			View:             planView,
+			ProviderSource:   providerSource,
+		},
 	}
 
 	args := []string{
 		"-out=terraform.plan",
 	}
+	code := pc.Run(args)
+	planOutput := planDone(t)
 
-	if code := pc.Run(args); code != 0 {
+	if code != 0 {
-		fmt.Println(ui.OutputWriter.String())
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, planOutput.Stderr())
-		t.Fatalf("wrong exit status %d; want 0\nstderr: %s", code, ui.ErrorWriter.String())
 	}
 
-	// flush the plan output from the mock ui
+	// show
-	ui.OutputWriter.Reset()
+	showView, showDone := testView(t)
 	sc := &ShowCommand{
-		Meta: m,
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(p),
+			View:             showView,
+			ProviderSource:   providerSource,
+		},
 	}
 
 	args = []string{
@@ -465,25 +643,27 @@ func TestShow_json_output_sensitive(t *testing.T) {
 		"terraform.plan",
 	}
 	defer os.Remove("terraform.plan")
+	code = sc.Run(args)
+	showOutput := showDone(t)
 
-	if code := sc.Run(args); code != 0 {
+	if code != 0 {
-		t.Fatalf("wrong exit status %d; want 0\nstderr: %s", code, ui.ErrorWriter.String())
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, showOutput.Stderr())
 	}
 
 	// compare ui output to wanted output
 	var got, want plan
 
-	gotString := ui.OutputWriter.String()
+	gotString := showOutput.Stdout()
 	json.Unmarshal([]byte(gotString), &got)
 
 	wantFile, err := os.Open("output.json")
 	if err != nil {
-		t.Fatalf("err: %s", err)
+		t.Fatalf("unexpected err: %s", err)
 	}
 	defer wantFile.Close()
 	byteValue, err := ioutil.ReadAll(wantFile)
 	if err != nil {
-		t.Fatalf("err: %s", err)
+		t.Fatalf("unexpected err: %s", err)
 	}
 	json.Unmarshal([]byte(byteValue), &want)
 
@@ -518,31 +698,35 @@ func TestShow_json_output_state(t *testing.T) {
 			defer close()
 
 			p := showFixtureProvider()
-			ui := new(cli.MockUi)
-			view, _ := testView(t)
-			m := Meta{
-				testingOverrides: metaOverridesForProvider(p),
-				Ui:               ui,
-				View:             view,
-				ProviderSource:   providerSource,
-			}
 
 			// init
+			ui := new(cli.MockUi)
 			ic := &InitCommand{
-				Meta: m,
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					Ui:               ui,
+					ProviderSource:   providerSource,
+				},
 			}
 			if code := ic.Run([]string{}); code != 0 {
 				t.Fatalf("init failed\n%s", ui.ErrorWriter)
 			}
 
-			// flush the plan output from the mock ui
+			// show
-			ui.OutputWriter.Reset()
+			showView, showDone := testView(t)
 			sc := &ShowCommand{
-				Meta: m,
+				Meta: Meta{
+					testingOverrides: metaOverridesForProvider(p),
+					View:             showView,
+					ProviderSource:   providerSource,
+				},
 			}
 
-			if code := sc.Run([]string{"-json"}); code != 0 {
+			code := sc.Run([]string{"-json"})
-				t.Fatalf("wrong exit status %d; want 0\nstderr: %s", code, ui.ErrorWriter.String())
+			showOutput := showDone(t)
+
+			if code != 0 {
+				t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, showOutput.Stderr())
 			}
 
 			// compare ui output to wanted output
@@ -554,17 +738,17 @@ func TestShow_json_output_state(t *testing.T) {
 			}
 			var got, want state
 
-			gotString := ui.OutputWriter.String()
+			gotString := showOutput.Stdout()
 			json.Unmarshal([]byte(gotString), &got)
 
 			wantFile, err := os.Open("output.json")
 			if err != nil {
-				t.Fatalf("err: %s", err)
+				t.Fatalf("unexpected error: %s", err)
 			}
 			defer wantFile.Close()
 			byteValue, err := ioutil.ReadAll(wantFile)
 			if err != nil {
-				t.Fatalf("err: %s", err)
+				t.Fatalf("unexpected err: %s", err)
 			}
 			json.Unmarshal([]byte(byteValue), &want)
 
@@ -575,6 +759,54 @@ func TestShow_json_output_state(t *testing.T) {
 	}
 }
 
+func TestShow_planWithNonDefaultStateLineage(t *testing.T) {
+	// Create a temporary working directory that is empty
+	td := tempDir(t)
+	testCopyDir(t, testFixturePath("show"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	// Write default state file with a testing lineage ("fake-for-testing")
+	testStateFileDefault(t, testState())
+
+	// Create a plan with a different lineage, which we should still be able
+	// to show
+	_, snap := testModuleWithSnapshot(t, "show")
+	state := testState()
+	plan := testPlan(t)
+	stateMeta := statemgr.SnapshotMeta{
+		Lineage:          "fake-for-plan",
+		Serial:           1,
+		TerraformVersion: version.SemVer,
+	}
+	planPath := testPlanFileMatchState(t, snap, state, plan, stateMeta)
+
+	view, done := testView(t)
+	c := &ShowCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			View:             view,
+		},
+	}
+
+	args := []string{
+		planPath,
+		"-no-color",
+	}
+	code := c.Run(args)
+	output := done(t)
+
+	if code != 0 {
+		t.Fatalf("unexpected exit status %d; want 0\ngot: %s", code, output.Stderr())
+	}
+
+	got := output.Stdout()
+	want := `No changes. Your infrastructure matches the configuration.`
+	if !strings.Contains(got, want) {
+		t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
+	}
+}
+
 // showFixtureSchema returns a schema suitable for processing the configuration
 // in testdata/show. This schema should be assigned to a mock provider
 // named "test".

internal/command/testdata/show-json-state/no-state/output.json

@@ -0,0 +1,3 @@
+{
+    "format_version": "1.0"
+}

internal/command/views/show.go

@@ -2,37 +2,95 @@ package views
 
 import (
 	"fmt"
-
 	"github.com/hashicorp/terraform/internal/command/arguments"
+	"github.com/hashicorp/terraform/internal/command/format"
+	"github.com/hashicorp/terraform/internal/command/jsonplan"
+	"github.com/hashicorp/terraform/internal/command/jsonstate"
+	"github.com/hashicorp/terraform/internal/configs"
 	"github.com/hashicorp/terraform/internal/plans"
+	"github.com/hashicorp/terraform/internal/states/statefile"
 	"github.com/hashicorp/terraform/internal/terraform"
+	"github.com/hashicorp/terraform/internal/tfdiags"
 )
 
-// FIXME: this is a temporary partial definition of the view for the show
-// command, in place to allow access to the plan renderer which is now in the
-// views package.
 type Show interface {
-	Plan(plan *plans.Plan, schemas *terraform.Schemas)
+	// Display renders the plan, if it is available. If plan is nil, it renders the statefile.
+	Display(config *configs.Config, plan *plans.Plan, stateFile *statefile.File, schemas *terraform.Schemas) int
+
+	// Diagnostics renders early diagnostics, resulting from argument parsing.
+	Diagnostics(diags tfdiags.Diagnostics)
 }
 
-// FIXME: the show view should support both human and JSON types. This code is
-// currently only used to render the plan in human-readable UI, so does not yet
-// support JSON.
 func NewShow(vt arguments.ViewType, view *View) Show {
 	switch vt {
+	case arguments.ViewJSON:
+		return &ShowJSON{view: view}
 	case arguments.ViewHuman:
-		return &ShowHuman{View: *view}
+		return &ShowHuman{view: view}
 	default:
 		panic(fmt.Sprintf("unknown view type %v", vt))
 	}
 }
 
 type ShowHuman struct {
-	View
+	view *View
 }
 
 var _ Show = (*ShowHuman)(nil)
 
-func (v *ShowHuman) Plan(plan *plans.Plan, schemas *terraform.Schemas) {
+func (v *ShowHuman) Display(config *configs.Config, plan *plans.Plan, stateFile *statefile.File, schemas *terraform.Schemas) int {
-	renderPlan(plan, schemas, &v.View)
+	if plan != nil {
+		renderPlan(plan, schemas, v.view)
+	} else {
+		if stateFile == nil {
+			v.view.streams.Println("No state.")
+			return 0
+		}
+
+		v.view.streams.Println(format.State(&format.StateOpts{
+			State:   stateFile.State,
+			Color:   v.view.colorize,
+			Schemas: schemas,
+		}))
+	}
+	return 0
+}
+
+func (v *ShowHuman) Diagnostics(diags tfdiags.Diagnostics) {
+	v.view.Diagnostics(diags)
+}
+
+type ShowJSON struct {
+	view *View
+}
+
+var _ Show = (*ShowJSON)(nil)
+
+func (v *ShowJSON) Display(config *configs.Config, plan *plans.Plan, stateFile *statefile.File, schemas *terraform.Schemas) int {
+	if plan != nil {
+		jsonPlan, err := jsonplan.Marshal(config, plan, stateFile, schemas)
+
+		if err != nil {
+			v.view.streams.Eprintf("Failed to marshal plan to json: %s", err)
+			return 1
+		}
+		v.view.streams.Println(string(jsonPlan))
+	} else {
+		// It is possible that there is neither state nor a plan.
+		// That's ok, we'll just return an empty object.
+		jsonState, err := jsonstate.Marshal(stateFile, schemas)
+		if err != nil {
+			v.view.streams.Eprintf("Failed to marshal state to json: %s", err)
+			return 1
+		}
+		v.view.streams.Println(string(jsonState))
+	}
+	return 0
+}
+
+// Diagnostics should only be called if show cannot be executed.
+// In this case, we choose to render human-readable diagnostic output,
+// primarily for backwards compatibility.
+func (v *ShowJSON) Diagnostics(diags tfdiags.Diagnostics) {
+	v.view.Diagnostics(diags)
 }

internal/command/views/show_test.go

@@ -0,0 +1,184 @@
+package views
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/terraform/internal/addrs"
+	"github.com/hashicorp/terraform/internal/command/arguments"
+	"github.com/hashicorp/terraform/internal/configs/configschema"
+	"github.com/hashicorp/terraform/internal/initwd"
+	"github.com/hashicorp/terraform/internal/plans"
+	"github.com/hashicorp/terraform/internal/states"
+	"github.com/hashicorp/terraform/internal/states/statefile"
+	"github.com/hashicorp/terraform/internal/terminal"
+	"github.com/hashicorp/terraform/internal/terraform"
+
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestShowHuman(t *testing.T) {
+	testCases := map[string]struct {
+		plan       *plans.Plan
+		stateFile  *statefile.File
+		schemas    *terraform.Schemas
+		wantExact  bool
+		wantString string
+	}{
+		"plan file": {
+			testPlan(t),
+			nil,
+			testSchemas(),
+			false,
+			"# test_resource.foo will be created",
+		},
+		"statefile": {
+			nil,
+			&statefile.File{
+				Serial:  0,
+				Lineage: "fake-for-testing",
+				State:   testState(),
+			},
+			testSchemas(),
+			false,
+			"# test_resource.foo:",
+		},
+		"empty statefile": {
+			nil,
+			&statefile.File{
+				Serial:  0,
+				Lineage: "fake-for-testing",
+				State:   states.NewState(),
+			},
+			testSchemas(),
+			true,
+			"\n",
+		},
+		"nothing": {
+			nil,
+			nil,
+			nil,
+			true,
+			"No state.\n",
+		},
+	}
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			streams, done := terminal.StreamsForTesting(t)
+			view := NewView(streams)
+			view.Configure(&arguments.View{NoColor: true})
+			v := NewShow(arguments.ViewHuman, view)
+
+			code := v.Display(nil, testCase.plan, testCase.stateFile, testCase.schemas)
+			if code != 0 {
+				t.Errorf("expected 0 return code, got %d", code)
+			}
+
+			output := done(t)
+			got := output.Stdout()
+			want := testCase.wantString
+			if (testCase.wantExact && got != want) || (!testCase.wantExact && !strings.Contains(got, want)) {
+				t.Fatalf("unexpected output\ngot: %s\nwant: %s", got, want)
+			}
+		})
+	}
+}
+
+func TestShowJSON(t *testing.T) {
+	testCases := map[string]struct {
+		plan      *plans.Plan
+		stateFile *statefile.File
+	}{
+		"plan file": {
+			testPlan(t),
+			nil,
+		},
+		"statefile": {
+			nil,
+			&statefile.File{
+				Serial:  0,
+				Lineage: "fake-for-testing",
+				State:   testState(),
+			},
+		},
+		"empty statefile": {
+			nil,
+			&statefile.File{
+				Serial:  0,
+				Lineage: "fake-for-testing",
+				State:   states.NewState(),
+			},
+		},
+		"nothing": {
+			nil,
+			nil,
+		},
+	}
+
+	config, _, configCleanup := initwd.MustLoadConfigForTests(t, "./testdata/show")
+	defer configCleanup()
+
+	for name, testCase := range testCases {
+		t.Run(name, func(t *testing.T) {
+			streams, done := terminal.StreamsForTesting(t)
+			view := NewView(streams)
+			view.Configure(&arguments.View{NoColor: true})
+			v := NewShow(arguments.ViewJSON, view)
+
+			schemas := &terraform.Schemas{
+				Providers: map[addrs.Provider]*terraform.ProviderSchema{
+					addrs.NewDefaultProvider("test"): {
+						ResourceTypes: map[string]*configschema.Block{
+							"test_resource": {
+								Attributes: map[string]*configschema.Attribute{
+									"id":  {Type: cty.String, Optional: true, Computed: true},
+									"foo": {Type: cty.String, Optional: true},
+								},
+							},
+						},
+					},
+				},
+			}
+
+			code := v.Display(config, testCase.plan, testCase.stateFile, schemas)
+
+			if code != 0 {
+				t.Errorf("expected 0 return code, got %d", code)
+			}
+
+			// Make sure the result looks like JSON; we comprehensively test
+			// the structure of this output in the command package tests.
+			var result map[string]interface{}
+			got := done(t).All()
+			t.Logf("output: %s", got)
+			if err := json.Unmarshal([]byte(got), &result); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// testState returns a test State structure.
+func testState() *states.State {
+	return states.BuildState(func(s *states.SyncState) {
+		s.SetResourceInstanceCurrent(
+			addrs.Resource{
+				Mode: addrs.ManagedResourceMode,
+				Type: "test_resource",
+				Name: "foo",
+			}.Instance(addrs.NoKey).Absolute(addrs.RootModuleInstance),
+			&states.ResourceInstanceObjectSrc{
+				AttrsJSON: []byte(`{"id":"bar","foo":"value"}`),
+				Status:    states.ObjectReady,
+			},
+			addrs.AbsProviderConfig{
+				Provider: addrs.NewDefaultProvider("test"),
+				Module:   addrs.RootModule,
+			},
+		)
+		// DeepCopy is used here to ensure our synthetic state matches exactly
+		// with a state that will have been copied during the command
+		// operation, and all fields have been copied correctly.
+	}).DeepCopy()
+}

internal/command/views/testdata/show/main.tf

@@ -0,0 +1,3 @@
+resource "test_resource" "foo" {
+    foo = "value"
+}

internal/command/workspace_delete.go

@@ -190,7 +190,6 @@ func (c *WorkspaceDeleteCommand) Run(args []string) int {
 
 func (c *WorkspaceDeleteCommand) AutocompleteArgs() complete.Predictor {
 	return completePredictSequence{
-		complete.PredictNothing, // the "select" subcommand itself (already matched)
 		c.completePredictWorkspaceName(),
 		complete.PredictDirs(""),
 	}

internal/command/workspace_new.go

@@ -167,7 +167,6 @@ func (c *WorkspaceNewCommand) Run(args []string) int {
 
 func (c *WorkspaceNewCommand) AutocompleteArgs() complete.Predictor {
 	return completePredictSequence{
-		complete.PredictNothing, // the "new" subcommand itself (already matched)
 		complete.PredictAnything,
 		complete.PredictDirs(""),
 	}

internal/command/workspace_select.go

@@ -117,7 +117,6 @@ func (c *WorkspaceSelectCommand) Run(args []string) int {
 
 func (c *WorkspaceSelectCommand) AutocompleteArgs() complete.Predictor {
 	return completePredictSequence{
-		complete.PredictNothing, // the "select" subcommand itself (already matched)
 		c.completePredictWorkspaceName(),
 		complete.PredictDirs(""),
 	}

internal/configs/config_build.go

@@ -23,9 +23,13 @@ func BuildConfig(root *Module, walker ModuleWalker) (*Config, hcl.Diagnostics) {
 	cfg.Root = cfg // Root module is self-referential.
 	cfg.Children, diags = buildChildModules(cfg, walker)
 
+	// Skip provider resolution if there are any errors, since the provider
+	// configurations themselves may not be valid.
+	if !diags.HasErrors() {
 		// Now that the config is built, we can connect the provider names to all
 		// the known types for validation.
 		cfg.resolveProviderTypes()
+	}
 
 	diags = append(diags, validateProviderConfigs(nil, cfg, false)...)
 

internal/configs/testdata/config-diagnostics/invalid-provider/errors

@@ -0,0 +1 @@
+main.tf:1,1-20: Invalid provider local name; crash_es is an invalid provider local name

internal/configs/testdata/config-diagnostics/invalid-provider/main.tf

@@ -0,0 +1,3 @@
+module "mod" {
+  source = "./mod"
+}

internal/configs/testdata/config-diagnostics/invalid-provider/mod/main.tf

@@ -0,0 +1,2 @@
+provider "crash_es" {
+}

internal/configs/testdata/config-diagnostics/invalid-provider/warnings

@@ -0,0 +1 @@
+Empty provider configuration blocks are not required

internal/dag/dag.go

@@ -2,7 +2,6 @@ package dag
 
 import (
 	"fmt"
-	"sort"
 	"strings"
 
 	"github.com/hashicorp/terraform/internal/tfdiags"
@@ -89,9 +88,7 @@ func (g *AcyclicGraph) Root() (Vertex, error) {
 // same graph with only a single edge between A and B, and a single edge
 // between B and C.
 //
-// The graph must be valid for this operation to behave properly. If
+// The graph must be free of cycles for this operation to behave properly.
-// Validate() returns an error, the behavior is undefined and the results
-// will likely be unexpected.
 //
 // Complexity: O(V(V+E)), or asymptotically O(VE)
 func (g *AcyclicGraph) TransitiveReduction() {
@@ -146,6 +143,8 @@ func (g *AcyclicGraph) Validate() error {
 	return err
 }
 
+// Cycles reports any cycles between graph nodes.
+// Self-referencing nodes are not reported, and must be detected separately.
 func (g *AcyclicGraph) Cycles() [][]Vertex {
 	var cycles [][]Vertex
 	for _, cycle := range StronglyConnected(&g.Graph) {
@@ -181,6 +180,8 @@ type vertexAtDepth struct {
 
 // DepthFirstWalk does a depth-first walk of the graph starting from
 // the vertices in start.
+// The algorithm used here does not do a complete topological sort. To ensure
+// correct overall ordering run TransitiveReduction first.
 func (g *AcyclicGraph) DepthFirstWalk(start Set, f DepthWalkFunc) error {
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, 0, len(start))
@@ -218,51 +219,10 @@ func (g *AcyclicGraph) DepthFirstWalk(start Set, f DepthWalkFunc) error {
 	return nil
 }
 
-// SortedDepthFirstWalk does a depth-first walk of the graph starting from
-// the vertices in start, always iterating the nodes in a consistent order.
-func (g *AcyclicGraph) SortedDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
-	seen := make(map[Vertex]struct{})
-	frontier := make([]*vertexAtDepth, len(start))
-	for i, v := range start {
-		frontier[i] = &vertexAtDepth{
-			Vertex: v,
-			Depth:  0,
-		}
-	}
-	for len(frontier) > 0 {
-		// Pop the current vertex
-		n := len(frontier)
-		current := frontier[n-1]
-		frontier = frontier[:n-1]
-
-		// Check if we've seen this already and return...
-		if _, ok := seen[current.Vertex]; ok {
-			continue
-		}
-		seen[current.Vertex] = struct{}{}
-
-		// Visit the current node
-		if err := f(current.Vertex, current.Depth); err != nil {
-			return err
-		}
-
-		// Visit targets of this in a consistent order.
-		targets := AsVertexList(g.downEdgesNoCopy(current.Vertex))
-		sort.Sort(byVertexName(targets))
-
-		for _, t := range targets {
-			frontier = append(frontier, &vertexAtDepth{
-				Vertex: t,
-				Depth:  current.Depth + 1,
-			})
-		}
-	}
-
-	return nil
-}
-
 // ReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
 // the vertices in start.
+// The algorithm used here does not do a complete topological sort. To ensure
+// correct overall ordering run TransitiveReduction first.
 func (g *AcyclicGraph) ReverseDepthFirstWalk(start Set, f DepthWalkFunc) error {
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, 0, len(start))
@@ -299,55 +259,3 @@ func (g *AcyclicGraph) ReverseDepthFirstWalk(start Set, f DepthWalkFunc) error {
 
 	return nil
 }
-
-// SortedReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
-// the vertices in start, always iterating the nodes in a consistent order.
-func (g *AcyclicGraph) SortedReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
-	seen := make(map[Vertex]struct{})
-	frontier := make([]*vertexAtDepth, len(start))
-	for i, v := range start {
-		frontier[i] = &vertexAtDepth{
-			Vertex: v,
-			Depth:  0,
-		}
-	}
-	for len(frontier) > 0 {
-		// Pop the current vertex
-		n := len(frontier)
-		current := frontier[n-1]
-		frontier = frontier[:n-1]
-
-		// Check if we've seen this already and return...
-		if _, ok := seen[current.Vertex]; ok {
-			continue
-		}
-		seen[current.Vertex] = struct{}{}
-
-		// Add next set of targets in a consistent order.
-		targets := AsVertexList(g.upEdgesNoCopy(current.Vertex))
-		sort.Sort(byVertexName(targets))
-		for _, t := range targets {
-			frontier = append(frontier, &vertexAtDepth{
-				Vertex: t,
-				Depth:  current.Depth + 1,
-			})
-		}
-
-		// Visit the current node
-		if err := f(current.Vertex, current.Depth); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// byVertexName implements sort.Interface so a list of Vertices can be sorted
-// consistently by their VertexName
-type byVertexName []Vertex
-
-func (b byVertexName) Len() int      { return len(b) }
-func (b byVertexName) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
-func (b byVertexName) Less(i, j int) bool {
-	return VertexName(b[i]) < VertexName(b[j])
-}

internal/dag/dag_test.go

@@ -99,6 +99,38 @@ func TestAyclicGraphTransReduction_more(t *testing.T) {
 	}
 }
 
+func TestAyclicGraphTransReduction_multipleRoots(t *testing.T) {
+	var g AcyclicGraph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Add(4)
+	g.Connect(BasicEdge(1, 2))
+	g.Connect(BasicEdge(1, 3))
+	g.Connect(BasicEdge(1, 4))
+	g.Connect(BasicEdge(2, 3))
+	g.Connect(BasicEdge(2, 4))
+	g.Connect(BasicEdge(3, 4))
+
+	g.Add(5)
+	g.Add(6)
+	g.Add(7)
+	g.Add(8)
+	g.Connect(BasicEdge(5, 6))
+	g.Connect(BasicEdge(5, 7))
+	g.Connect(BasicEdge(5, 8))
+	g.Connect(BasicEdge(6, 7))
+	g.Connect(BasicEdge(6, 8))
+	g.Connect(BasicEdge(7, 8))
+	g.TransitiveReduction()
+
+	actual := strings.TrimSpace(g.String())
+	expected := strings.TrimSpace(testGraphTransReductionMultipleRootsStr)
+	if actual != expected {
+		t.Fatalf("bad: %s", actual)
+	}
+}
+
 // use this to simulate slow sort operations
 type counter struct {
 	Name  string
@@ -392,7 +424,10 @@ func TestAcyclicGraph_ReverseDepthFirstWalk_WithRemoval(t *testing.T) {
 
 	var visits []Vertex
 	var lock sync.Mutex
-	err := g.SortedReverseDepthFirstWalk([]Vertex{1}, func(v Vertex, d int) error {
+	root := make(Set)
+	root.Add(1)
+
+	err := g.ReverseDepthFirstWalk(root, func(v Vertex, d int) error {
 		lock.Lock()
 		defer lock.Unlock()
 		visits = append(visits, v)
@@ -426,3 +461,20 @@ const testGraphTransReductionMoreStr = `
   4
 4
 `
+
+const testGraphTransReductionMultipleRootsStr = `
+1
+  2
+2
+  3
+3
+  4
+4
+5
+  6
+6
+  7
+7
+  8
+8
+`

internal/dag/edge.go

@@ -1,9 +1,5 @@
 package dag
 
-import (
-	"fmt"
-)
-
 // Edge represents an edge in the graph, with a source and target vertex.
 type Edge interface {
 	Source() Vertex
@@ -25,7 +21,7 @@ type basicEdge struct {
 }
 
 func (e *basicEdge) Hashcode() interface{} {
-	return fmt.Sprintf("%p-%p", e.S, e.T)
+	return [...]interface{}{e.S, e.T}
 }
 
 func (e *basicEdge) Source() Vertex {

internal/getmodules/getter.go

@@ -73,7 +73,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{
 var goGetterGetters = map[string]getter.Getter{
 	"file":  new(getter.FileGetter),
 	"gcs":   new(getter.GCSGetter),
-	"git":   new(gitGetter),
+	"git":   new(getter.GitGetter),
 	"hg":    new(getter.HgGetter),
 	"s3":    new(getter.S3Getter),
 	"http":  getterHTTPGetter,

internal/getmodules/git_getter.go

@@ -1,416 +0,0 @@
-package getmodules
-
-import (
-	"bytes"
-	"context"
-	"encoding/base64"
-	"fmt"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"strconv"
-	"strings"
-	"syscall"
-
-	getter "github.com/hashicorp/go-getter"
-	urlhelper "github.com/hashicorp/go-getter/helper/url"
-	safetemp "github.com/hashicorp/go-safetemp"
-	version "github.com/hashicorp/go-version"
-)
-
-// getter is our base getter; it regroups
-// fields all getters have in common.
-type getterCommon struct {
-	client *getter.Client
-}
-
-func (g *getterCommon) SetClient(c *getter.Client) { g.client = c }
-
-// Context tries to returns the Contex from the getter's
-// client. otherwise context.Background() is returned.
-func (g *getterCommon) Context() context.Context {
-	if g == nil || g.client == nil {
-		return context.Background()
-	}
-	return g.client.Ctx
-}
-
-// gitGetter is a temporary fork of getter.GitGetter to allow us to tactically
-// fix https://github.com/hashicorp/terraform/issues/30119 only within
-// Terraform.
-//
-// This should be only a brief workaround to help us decouple work on the
-// Terraform CLI v1.1.1 release so that we can get it done without having to
-// coordinate with every other go-getter caller first. However, this fork
-// should be healed promptly after v1.1.1 by upstreaming something like this
-// fix into upstream go-getter, so that other go-getter callers can also
-// benefit from it.
-type gitGetter struct {
-	getterCommon
-}
-
-var defaultBranchRegexp = regexp.MustCompile(`\s->\sorigin/(.*)`)
-var lsRemoteSymRefRegexp = regexp.MustCompile(`ref: refs/heads/([^\s]+).*`)
-
-func (g *gitGetter) ClientMode(_ *url.URL) (getter.ClientMode, error) {
-	return getter.ClientModeDir, nil
-}
-
-func (g *gitGetter) Get(dst string, u *url.URL) error {
-	ctx := g.Context()
-	if _, err := exec.LookPath("git"); err != nil {
-		return fmt.Errorf("git must be available and on the PATH")
-	}
-
-	// The port number must be parseable as an integer. If not, the user
-	// was probably trying to use a scp-style address, in which case the
-	// ssh:// prefix must be removed to indicate that.
-	//
-	// This is not necessary in versions of Go which have patched
-	// CVE-2019-14809 (e.g. Go 1.12.8+)
-	if portStr := u.Port(); portStr != "" {
-		if _, err := strconv.ParseUint(portStr, 10, 16); err != nil {
-			return fmt.Errorf("invalid port number %q; if using the \"scp-like\" git address scheme where a colon introduces the path instead, remove the ssh:// portion and use just the git:: prefix", portStr)
-		}
-	}
-
-	// Extract some query parameters we use
-	var ref, sshKey string
-	depth := 0 // 0 means "not set"
-	q := u.Query()
-	if len(q) > 0 {
-		ref = q.Get("ref")
-		q.Del("ref")
-
-		sshKey = q.Get("sshkey")
-		q.Del("sshkey")
-
-		if n, err := strconv.Atoi(q.Get("depth")); err == nil {
-			depth = n
-		}
-		q.Del("depth")
-
-		// Copy the URL
-		var newU url.URL = *u
-		u = &newU
-		u.RawQuery = q.Encode()
-	}
-
-	var sshKeyFile string
-	if sshKey != "" {
-		// Check that the git version is sufficiently new.
-		if err := checkGitVersion("2.3"); err != nil {
-			return fmt.Errorf("Error using ssh key: %v", err)
-		}
-
-		// We have an SSH key - decode it.
-		raw, err := base64.StdEncoding.DecodeString(sshKey)
-		if err != nil {
-			return err
-		}
-
-		// Create a temp file for the key and ensure it is removed.
-		fh, err := ioutil.TempFile("", "go-getter")
-		if err != nil {
-			return err
-		}
-		sshKeyFile = fh.Name()
-		defer os.Remove(sshKeyFile)
-
-		// Set the permissions prior to writing the key material.
-		if err := os.Chmod(sshKeyFile, 0600); err != nil {
-			return err
-		}
-
-		// Write the raw key into the temp file.
-		_, err = fh.Write(raw)
-		fh.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Clone or update the repository
-	_, err := os.Stat(dst)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	if err == nil {
-		err = g.update(ctx, dst, sshKeyFile, ref, depth)
-	} else {
-		err = g.clone(ctx, dst, sshKeyFile, u, ref, depth)
-	}
-	if err != nil {
-		return err
-	}
-
-	// Next: check out the proper tag/branch if it is specified, and checkout
-	if ref != "" {
-		if err := g.checkout(dst, ref); err != nil {
-			return err
-		}
-	}
-
-	// Lastly, download any/all submodules.
-	return g.fetchSubmodules(ctx, dst, sshKeyFile, depth)
-}
-
-// GetFile for Git doesn't support updating at this time. It will download
-// the file every time.
-func (g *gitGetter) GetFile(dst string, u *url.URL) error {
-	td, tdcloser, err := safetemp.Dir("", "getter")
-	if err != nil {
-		return err
-	}
-	defer tdcloser.Close()
-
-	// Get the filename, and strip the filename from the URL so we can
-	// just get the repository directly.
-	filename := filepath.Base(u.Path)
-	u.Path = filepath.Dir(u.Path)
-
-	// Get the full repository
-	if err := g.Get(td, u); err != nil {
-		return err
-	}
-
-	// Copy the single file
-	u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename)))
-	if err != nil {
-		return err
-	}
-
-	fg := &getter.FileGetter{Copy: true}
-	return fg.GetFile(dst, u)
-}
-
-func (g *gitGetter) checkout(dst string, ref string) error {
-	cmd := exec.Command("git", "checkout", ref)
-	cmd.Dir = dst
-	return getRunCommand(cmd)
-}
-
-// gitCommitIDRegex is a pattern intended to match strings that seem
-// "likely to be" git commit IDs, rather than named refs. This cannot be
-// an exact decision because it's valid to name a branch or tag after a series
-// of hexadecimal digits too.
-//
-// We require at least 7 digits here because that's the smallest size git
-// itself will typically generate, and so it'll reduce the risk of false
-// positives on short branch names that happen to also be "hex words".
-var gitCommitIDRegex = regexp.MustCompile("^[0-9a-fA-F]{7,40}$")
-
-func (g *gitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int) error {
-	args := []string{"clone"}
-
-	autoBranch := false
-	if ref == "" {
-		ref = findRemoteDefaultBranch(u)
-		autoBranch = true
-	}
-	if depth > 0 {
-		args = append(args, "--depth", strconv.Itoa(depth))
-		args = append(args, "--branch", ref)
-	}
-	args = append(args, u.String(), dst)
-
-	cmd := exec.CommandContext(ctx, "git", args...)
-	setupGitEnv(cmd, sshKeyFile)
-	err := getRunCommand(cmd)
-	if err != nil {
-		if depth > 0 && !autoBranch {
-			// If we're creating a shallow clone then the given ref must be
-			// a named ref (branch or tag) rather than a commit directly.
-			// We can't accurately recognize the resulting error here without
-			// hard-coding assumptions about git's human-readable output, but
-			// we can at least try a heuristic.
-			if gitCommitIDRegex.MatchString(ref) {
-				return fmt.Errorf("%w (note that setting 'depth' requires 'ref' to be a branch or tag name)", err)
-			}
-		}
-		return err
-	}
-
-	if depth < 1 && !autoBranch {
-		// If we didn't add --depth and --branch above then we will now be
-		// on the remote repository's default branch, rather than the selected
-		// ref, so we'll need to fix that before we return.
-		return g.checkout(dst, ref)
-	}
-	return nil
-}
-
-func (g *gitGetter) update(ctx context.Context, dst, sshKeyFile, ref string, depth int) error {
-	// Determine if we're a branch. If we're NOT a branch, then we just
-	// switch to master prior to checking out
-	cmd := exec.CommandContext(ctx, "git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
-	cmd.Dir = dst
-
-	if getRunCommand(cmd) != nil {
-		// Not a branch, switch to default branch. This will also catch
-		// non-existent branches, in which case we want to switch to default
-		// and then checkout the proper branch later.
-		ref = findDefaultBranch(dst)
-	}
-
-	// We have to be on a branch to pull
-	if err := g.checkout(dst, ref); err != nil {
-		return err
-	}
-
-	if depth > 0 {
-		cmd = exec.Command("git", "pull", "--depth", strconv.Itoa(depth), "--ff-only")
-	} else {
-		cmd = exec.Command("git", "pull", "--ff-only")
-	}
-
-	cmd.Dir = dst
-	setupGitEnv(cmd, sshKeyFile)
-	return getRunCommand(cmd)
-}
-
-// fetchSubmodules downloads any configured submodules recursively.
-func (g *gitGetter) fetchSubmodules(ctx context.Context, dst, sshKeyFile string, depth int) error {
-	args := []string{"submodule", "update", "--init", "--recursive"}
-	if depth > 0 {
-		args = append(args, "--depth", strconv.Itoa(depth))
-	}
-	cmd := exec.CommandContext(ctx, "git", args...)
-	cmd.Dir = dst
-	setupGitEnv(cmd, sshKeyFile)
-	return getRunCommand(cmd)
-}
-
-// findDefaultBranch checks the repo's origin remote for its default branch
-// (generally "master"). "master" is returned if an origin default branch
-// can't be determined.
-func findDefaultBranch(dst string) string {
-	var stdoutbuf bytes.Buffer
-	cmd := exec.Command("git", "branch", "-r", "--points-at", "refs/remotes/origin/HEAD")
-	cmd.Dir = dst
-	cmd.Stdout = &stdoutbuf
-	err := cmd.Run()
-	matches := defaultBranchRegexp.FindStringSubmatch(stdoutbuf.String())
-	if err != nil || matches == nil {
-		return "master"
-	}
-	return matches[len(matches)-1]
-}
-
-// findRemoteDefaultBranch checks the remote repo's HEAD symref to return the remote repo's
-// default branch. "master" is returned if no HEAD symref exists.
-func findRemoteDefaultBranch(u *url.URL) string {
-	var stdoutbuf bytes.Buffer
-	cmd := exec.Command("git", "ls-remote", "--symref", u.String(), "HEAD")
-	cmd.Stdout = &stdoutbuf
-	err := cmd.Run()
-	matches := lsRemoteSymRefRegexp.FindStringSubmatch(stdoutbuf.String())
-	if err != nil || matches == nil {
-		return "master"
-	}
-	return matches[len(matches)-1]
-}
-
-// setupGitEnv sets up the environment for the given command. This is used to
-// pass configuration data to git and ssh and enables advanced cloning methods.
-func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
-	const gitSSHCommand = "GIT_SSH_COMMAND="
-	var sshCmd []string
-
-	// If we have an existing GIT_SSH_COMMAND, we need to append our options.
-	// We will also remove our old entry to make sure the behavior is the same
-	// with versions of Go < 1.9.
-	env := os.Environ()
-	for i, v := range env {
-		if strings.HasPrefix(v, gitSSHCommand) && len(v) > len(gitSSHCommand) {
-			sshCmd = []string{v}
-
-			env[i], env[len(env)-1] = env[len(env)-1], env[i]
-			env = env[:len(env)-1]
-			break
-		}
-	}
-
-	if len(sshCmd) == 0 {
-		sshCmd = []string{gitSSHCommand + "ssh"}
-	}
-
-	if sshKeyFile != "" {
-		// We have an SSH key temp file configured, tell ssh about this.
-		if runtime.GOOS == "windows" {
-			sshKeyFile = strings.Replace(sshKeyFile, `\`, `/`, -1)
-		}
-		sshCmd = append(sshCmd, "-i", sshKeyFile)
-	}
-
-	env = append(env, strings.Join(sshCmd, " "))
-	cmd.Env = env
-}
-
-// checkGitVersion is used to check the version of git installed on the system
-// against a known minimum version. Returns an error if the installed version
-// is older than the given minimum.
-func checkGitVersion(min string) error {
-	want, err := version.NewVersion(min)
-	if err != nil {
-		return err
-	}
-
-	out, err := exec.Command("git", "version").Output()
-	if err != nil {
-		return err
-	}
-
-	fields := strings.Fields(string(out))
-	if len(fields) < 3 {
-		return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
-	}
-	v := fields[2]
-	if runtime.GOOS == "windows" && strings.Contains(v, ".windows.") {
-		// on windows, git version will return for example:
-		// git version 2.20.1.windows.1
-		// Which does not follow the semantic versionning specs
-		// https://semver.org. We remove that part in order for
-		// go-version to not error.
-		v = v[:strings.Index(v, ".windows.")]
-	}
-
-	have, err := version.NewVersion(v)
-	if err != nil {
-		return err
-	}
-
-	if have.LessThan(want) {
-		return fmt.Errorf("Required git version = %s, have %s", want, have)
-	}
-
-	return nil
-}
-
-// getRunCommand is a helper that will run a command and capture the output
-// in the case an error happens.
-func getRunCommand(cmd *exec.Cmd) error {
-	var buf bytes.Buffer
-	cmd.Stdout = &buf
-	cmd.Stderr = &buf
-	err := cmd.Run()
-	if err == nil {
-		return nil
-	}
-	if exiterr, ok := err.(*exec.ExitError); ok {
-		// The program has exited with an exit code != 0
-		if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-			return fmt.Errorf(
-				"%s exited with %d: %s",
-				cmd.Path,
-				status.ExitStatus(),
-				buf.String())
-		}
-	}
-
-	return fmt.Errorf("error running %s: %s", cmd.Path, buf.String())
-}

internal/getmodules/git_getter_test.go

@@ -1,827 +0,0 @@
-package getmodules
-
-import (
-	"bytes"
-	"encoding/base64"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"reflect"
-	"runtime"
-	"strings"
-	"testing"
-
-	getter "github.com/hashicorp/go-getter"
-	urlhelper "github.com/hashicorp/go-getter/helper/url"
-)
-
-var testHasGit bool
-
-func init() {
-	if _, err := exec.LookPath("git"); err == nil {
-		testHasGit = true
-	}
-}
-
-func TestGitGetter_impl(t *testing.T) {
-	var _ getter.Getter = new(gitGetter)
-}
-
-func TestGitGetter(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "basic")
-	repo.commitFile("foo.txt", "hello")
-
-	// With a dir that doesn't exist
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "foo.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_branch(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "branch")
-	repo.git("checkout", "-b", "test-branch")
-	repo.commitFile("branch.txt", "branch")
-
-	q := repo.url.Query()
-	q.Add("ref", "test-branch")
-	repo.url.RawQuery = q.Encode()
-
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "branch.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Get again should work
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath = filepath.Join(dst, "branch.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_commitID(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	// We're going to create different content on the main branch vs.
-	// another branch here, so that below we can recognize if we
-	// correctly cloned the commit actually requested (from the
-	// "other branch"), not the one at HEAD.
-	repo := testGitRepo(t, "commit_id")
-	repo.git("checkout", "-b", "main-branch")
-	repo.commitFile("wrong.txt", "Nope")
-	repo.git("checkout", "-b", "other-branch")
-	repo.commitFile("hello.txt", "Yep")
-	commitID, err := repo.latestCommit()
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Return to the main branch so that HEAD of this repository
-	// will be that, rather than "test-branch".
-	repo.git("checkout", "main-branch")
-
-	q := repo.url.Query()
-	q.Add("ref", commitID)
-	repo.url.RawQuery = q.Encode()
-
-	t.Logf("Getting %s", repo.url)
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "hello.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Get again should work
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath = filepath.Join(dst, "hello.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_remoteWithoutMaster(t *testing.T) {
-	if !testHasGit {
-		t.Log("git not found, skipping")
-		t.Skip()
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "branch")
-	repo.git("checkout", "-b", "test-branch")
-	repo.commitFile("branch.txt", "branch")
-
-	q := repo.url.Query()
-	repo.url.RawQuery = q.Encode()
-
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "branch.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Get again should work
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath = filepath.Join(dst, "branch.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_shallowClone(t *testing.T) {
-	if !testHasGit {
-		t.Log("git not found, skipping")
-		t.Skip()
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "upstream")
-	repo.commitFile("upstream.txt", "0")
-	repo.commitFile("upstream.txt", "1")
-
-	// Specifiy a clone depth of 1
-	q := repo.url.Query()
-	q.Add("depth", "1")
-	repo.url.RawQuery = q.Encode()
-
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Assert rev-list count is '1'
-	cmd := exec.Command("git", "rev-list", "HEAD", "--count")
-	cmd.Dir = dst
-	b, err := cmd.Output()
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	out := strings.TrimSpace(string(b))
-	if out != "1" {
-		t.Fatalf("expected rev-list count to be '1' but got %v", out)
-	}
-}
-
-func TestGitGetter_shallowCloneWithTag(t *testing.T) {
-	if !testHasGit {
-		t.Log("git not found, skipping")
-		t.Skip()
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "upstream")
-	repo.commitFile("v1.0.txt", "0")
-	repo.git("tag", "v1.0")
-	repo.commitFile("v1.1.txt", "1")
-
-	// Specifiy a clone depth of 1 with a tag
-	q := repo.url.Query()
-	q.Add("ref", "v1.0")
-	q.Add("depth", "1")
-	repo.url.RawQuery = q.Encode()
-
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Assert rev-list count is '1'
-	cmd := exec.Command("git", "rev-list", "HEAD", "--count")
-	cmd.Dir = dst
-	b, err := cmd.Output()
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	out := strings.TrimSpace(string(b))
-	if out != "1" {
-		t.Fatalf("expected rev-list count to be '1' but got %v", out)
-	}
-
-	// Verify the v1.0 file exists
-	mainPath := filepath.Join(dst, "v1.0.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the v1.1 file does not exists
-	mainPath = filepath.Join(dst, "v1.1.txt")
-	if _, err := os.Stat(mainPath); err == nil {
-		t.Fatalf("expected v1.1 file to not exist")
-	}
-}
-
-func TestGitGetter_shallowCloneWithCommitID(t *testing.T) {
-	if !testHasGit {
-		t.Log("git not found, skipping")
-		t.Skip()
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "upstream")
-	repo.commitFile("v1.0.txt", "0")
-	repo.git("tag", "v1.0")
-	repo.commitFile("v1.1.txt", "1")
-
-	commitID, err := repo.latestCommit()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Specify a clone depth of 1 with a naked commit ID
-	// This is intentionally invalid: shallow clone always requires a named ref.
-	q := repo.url.Query()
-	q.Add("ref", commitID[:8])
-	q.Add("depth", "1")
-	repo.url.RawQuery = q.Encode()
-
-	t.Logf("Getting %s", repo.url)
-	err = g.Get(dst, repo.url)
-	if err == nil {
-		t.Fatalf("success; want error")
-	}
-	// We use a heuristic to generate an extra hint in the error message if
-	// it looks like the user was trying to combine ref=COMMIT with depth.
-	if got, want := err.Error(), "(note that setting 'depth' requires 'ref' to be a branch or tag name)"; !strings.Contains(got, want) {
-		t.Errorf("missing error message hint\ngot: %s\nwant substring: %s", got, want)
-	}
-}
-
-func TestGitGetter_branchUpdate(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	// First setup the state with a fresh branch
-	repo := testGitRepo(t, "branch-update")
-	repo.git("checkout", "-b", "test-branch")
-	repo.commitFile("branch.txt", "branch")
-
-	// Get the "test-branch" branch
-	q := repo.url.Query()
-	q.Add("ref", "test-branch")
-	repo.url.RawQuery = q.Encode()
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "branch.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Commit an update to the branch
-	repo.commitFile("branch-update.txt", "branch-update")
-
-	// Get again should work
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath = filepath.Join(dst, "branch-update.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_tag(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	repo := testGitRepo(t, "tag")
-	repo.commitFile("tag.txt", "tag")
-	repo.git("tag", "v1.0")
-
-	q := repo.url.Query()
-	q.Add("ref", "v1.0")
-	repo.url.RawQuery = q.Encode()
-
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath := filepath.Join(dst, "tag.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Get again should work
-	if err := g.Get(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	mainPath = filepath.Join(dst, "tag.txt")
-	if _, err := os.Stat(mainPath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_GetFile(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempTestFile(t)
-	defer os.RemoveAll(filepath.Dir(dst))
-
-	repo := testGitRepo(t, "file")
-	repo.commitFile("file.txt", "hello")
-
-	// Download the file
-	repo.url.Path = filepath.Join(repo.url.Path, "file.txt")
-	if err := g.GetFile(dst, repo.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Verify the main file exists
-	if _, err := os.Stat(dst); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	assertContents(t, dst, "hello")
-}
-
-func TestGitGetter_gitVersion(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-	if runtime.GOOS == "windows" {
-		t.Skip("skipping on windows since the test requires sh")
-	}
-	dir, err := ioutil.TempDir("", "go-getter")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	script := filepath.Join(dir, "git")
-	err = ioutil.WriteFile(
-		script,
-		[]byte("#!/bin/sh\necho \"git version 2.0 (Some Metadata Here)\n\""),
-		0700)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer func(v string) {
-		os.Setenv("PATH", v)
-	}(os.Getenv("PATH"))
-
-	os.Setenv("PATH", dir)
-
-	// Asking for a higher version throws an error
-	if err := checkGitVersion("2.3"); err == nil {
-		t.Fatal("expect git version error")
-	}
-
-	// Passes when version is satisfied
-	if err := checkGitVersion("1.9"); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestGitGetter_sshKey(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
-
-	// avoid getting locked by a github authenticity validation prompt
-	os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
-	defer os.Setenv("GIT_SSH_COMMAND", "")
-
-	u, err := urlhelper.Parse("ssh://git@github.com/hashicorp/test-private-repo" +
-		"?sshkey=" + encodedKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := g.Get(dst, u); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	readmePath := filepath.Join(dst, "README.md")
-	if _, err := os.Stat(readmePath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_sshSCPStyle(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
-
-	// avoid getting locked by a github authenticity validation prompt
-	os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
-	defer os.Setenv("GIT_SSH_COMMAND", "")
-
-	// This test exercises the combination of the git detector and the
-	// git getter, to make sure that together they make scp-style URLs work.
-	client := &getter.Client{
-		Src: "git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey,
-		Dst: dst,
-		Pwd: ".",
-
-		Mode: getter.ClientModeDir,
-
-		Detectors: []getter.Detector{
-			new(getter.GitDetector),
-		},
-		Getters: map[string]getter.Getter{
-			"git": g,
-		},
-	}
-
-	if err := client.Get(); err != nil {
-		t.Fatalf("client.Get failed: %s", err)
-	}
-
-	readmePath := filepath.Join(dst, "README.md")
-	if _, err := os.Stat(readmePath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_sshExplicitPort(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
-
-	// avoid getting locked by a github authenticity validation prompt
-	os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
-	defer os.Setenv("GIT_SSH_COMMAND", "")
-
-	// This test exercises the combination of the git detector and the
-	// git getter, to make sure that together they make scp-style URLs work.
-	client := &getter.Client{
-		Src: "git::ssh://git@github.com:22/hashicorp/test-private-repo?sshkey=" + encodedKey,
-		Dst: dst,
-		Pwd: ".",
-
-		Mode: getter.ClientModeDir,
-
-		Detectors: []getter.Detector{
-			new(getter.GitDetector),
-		},
-		Getters: map[string]getter.Getter{
-			"git": g,
-		},
-	}
-
-	if err := client.Get(); err != nil {
-		t.Fatalf("client.Get failed: %s", err)
-	}
-
-	readmePath := filepath.Join(dst, "README.md")
-	if _, err := os.Stat(readmePath); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-}
-
-func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
-
-	// avoid getting locked by a github authenticity validation prompt
-	os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
-	defer os.Setenv("GIT_SSH_COMMAND", "")
-
-	// This test exercises the combination of the git detector and the
-	// git getter, to make sure that together they make scp-style URLs work.
-	client := &getter.Client{
-		Src: "git::ssh://git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey,
-		Dst: dst,
-		Pwd: ".",
-
-		Mode: getter.ClientModeDir,
-
-		Detectors: []getter.Detector{
-			new(getter.GitDetector),
-		},
-		Getters: map[string]getter.Getter{
-			"git": g,
-		},
-	}
-
-	err := client.Get()
-	if err == nil {
-		t.Fatalf("get succeeded; want error")
-	}
-
-	got := err.Error()
-	want1, want2 := `invalid source string`, `invalid port number "hashicorp"`
-	if !(strings.Contains(got, want1) || strings.Contains(got, want2)) {
-		t.Fatalf("wrong error\ngot:  %s\nwant: %q or %q", got, want1, want2)
-	}
-}
-
-func TestGitGetter_submodule(t *testing.T) {
-	if !testHasGit {
-		t.Skip("git not found, skipping")
-	}
-
-	g := new(gitGetter)
-	dst := tempDir(t)
-
-	relpath := func(basepath, targpath string) string {
-		relpath, err := filepath.Rel(basepath, targpath)
-		if err != nil {
-			t.Fatal(err)
-		}
-		return strings.Replace(relpath, `\`, `/`, -1)
-		// on windows git still prefers relatives paths
-		// containing `/` for submodules
-	}
-
-	// Set up the grandchild
-	gc := testGitRepo(t, "grandchild")
-	gc.commitFile("grandchild.txt", "grandchild")
-
-	// Set up the child
-	c := testGitRepo(t, "child")
-	c.commitFile("child.txt", "child")
-	c.git("submodule", "add", "-f", relpath(c.dir, gc.dir))
-	c.git("commit", "-m", "Add grandchild submodule")
-
-	// Set up the parent
-	p := testGitRepo(t, "parent")
-	p.commitFile("parent.txt", "parent")
-	p.git("submodule", "add", "-f", relpath(p.dir, c.dir))
-	p.git("commit", "-m", "Add child submodule")
-
-	// Clone the root repository
-	if err := g.Get(dst, p.url); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	// Check that the files exist
-	for _, path := range []string{
-		filepath.Join(dst, "parent.txt"),
-		filepath.Join(dst, "child", "child.txt"),
-		filepath.Join(dst, "child", "grandchild", "grandchild.txt"),
-	} {
-		if _, err := os.Stat(path); err != nil {
-			t.Fatalf("err: %s", err)
-		}
-	}
-}
-
-func TestGitGetter_setupGitEnv_sshKey(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("skipping on windows since the test requires sh")
-	}
-
-	cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND")
-	setupGitEnv(cmd, "/tmp/foo.pem")
-	out, err := cmd.Output()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	actual := strings.TrimSpace(string(out))
-	if actual != "ssh -i /tmp/foo.pem" {
-		t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual)
-	}
-}
-
-func TestGitGetter_setupGitEnvWithExisting_sshKey(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skipf("skipping on windows since the test requires sh")
-		return
-	}
-
-	// start with an existing ssh command configuration
-	os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
-	defer os.Setenv("GIT_SSH_COMMAND", "")
-
-	cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND")
-	setupGitEnv(cmd, "/tmp/foo.pem")
-	out, err := cmd.Output()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	actual := strings.TrimSpace(string(out))
-	if actual != "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i /tmp/foo.pem" {
-		t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual)
-	}
-}
-
-// gitRepo is a helper struct which controls a single temp git repo.
-type gitRepo struct {
-	t   *testing.T
-	url *url.URL
-	dir string
-}
-
-// testGitRepo creates a new test git repository.
-func testGitRepo(t *testing.T, name string) *gitRepo {
-	t.Helper()
-	dir, err := ioutil.TempDir("", "go-getter")
-	if err != nil {
-		t.Fatal(err)
-	}
-	dir = filepath.Join(dir, name)
-	if err := os.Mkdir(dir, 0700); err != nil {
-		t.Fatal(err)
-	}
-
-	r := &gitRepo{
-		t:   t,
-		dir: dir,
-	}
-
-	url, err := urlhelper.Parse("file://" + r.dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	r.url = url
-
-	t.Logf("initializing git repo in %s", dir)
-	r.git("init")
-	r.git("config", "user.name", "go-getter")
-	r.git("config", "user.email", "go-getter@hashicorp.com")
-
-	return r
-}
-
-// git runs a git command against the repo.
-func (r *gitRepo) git(args ...string) {
-	cmd := exec.Command("git", args...)
-	cmd.Dir = r.dir
-	bfr := bytes.NewBuffer(nil)
-	cmd.Stderr = bfr
-	if err := cmd.Run(); err != nil {
-		r.t.Fatal(err, bfr.String())
-	}
-}
-
-// commitFile writes and commits a text file to the repo.
-func (r *gitRepo) commitFile(file, content string) {
-	path := filepath.Join(r.dir, file)
-	if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil {
-		r.t.Fatal(err)
-	}
-	r.git("add", file)
-	r.git("commit", "-m", "Adding "+file)
-}
-
-// latestCommit returns the full commit id of the latest commit on the current
-// branch.
-func (r *gitRepo) latestCommit() (string, error) {
-	cmd := exec.Command("git", "rev-parse", "HEAD")
-	cmd.Dir = r.dir
-	rawOut, err := cmd.Output()
-	if err != nil {
-		return "", err
-	}
-	rawOut = bytes.TrimSpace(rawOut)
-	return string(rawOut), nil
-}
-
-// This is a read-only deploy key for an empty test repository.
-// Note: This is split over multiple lines to avoid being disabled by key
-// scanners automatically.
-var testGitToken = `-----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEA9cHsxCl3Jjgu9DHpwvmfFOl1XEdY+ShHDR/cMnzJ5ddk5/oV
-Wy6EWatvyHZfRSZMwzv4PtKeUPm6iXjqWp4xdWU9khlPzozyj+U9Fq70TRVUW9E5
-T1XdQVwJE421yffr4VMMwu60wBqjI1epapH2i2inYvw9Zl9X2MXq0+jTvFvDerbT
-mDtfStDPljenELAIZtWVETSvbI46gALwbxbM2292ZUIL4D6jRz0aZMmyy/twYv8r
-9WGJLwmYzU518Ie7zqKW/mCTdTrV0WRiDj0MeRaPgrGY9amuHE4r9iG/cJkwpKAO
-Ccz0Hs6i89u9vZnTqZU9V7weJqRAQcMjXXR6yQIDAQABAoIBAQDBzICKnGxiTlHw
-rd+6qqChnAy5jWYDbZjCJ8q8YZ3RS08+g/8NXZxvHftTqM0uOaq1FviHig3gq15H
-hHvCpBc6jXDFYoKFzq6FfO/0kFkE5HoWweIgxwRow0xBCDJAJ+ryUEyy+Ay/pQHb
-IAjwilRS0V+WdnVw4mTjBAhPvb4jPOo97Yfy3PYUyx2F3newkqXOZy+zx3G/ANoa
-ncypfMGyy76sfCWKqw4J1gVkVQLwbB6gQkXUFGYwY9sRrxbG93kQw76Flc/E/s52
-62j4v1IM0fq0t/St+Y/+s6Lkw` + `aqt3ft1nsqWcRaVDdqvMfkzgJGXlw0bGzJG5MEQ
-AIBq3dHRAoGBAP8OeG/DKG2Z1VmSfzuz1pas1fbZ+F7venOBrjez3sKlb3Pyl2aH
-mt2wjaTUi5v10VrHgYtOEdqyhQeUSYydWXIBKNMag0NLLrfFUKZK+57wrHWFdFjn
-VgpsdkLSNTOZpC8gA5OaJ+36IcOPfGqyyP9wuuRoaYnVT1KEzqLa9FEFAoGBAPaq
-pglwhil2rxjJE4zq0afQLNpAfi7Xqcrepij+xvJIcIj7nawxXuPxqRFxONE/h3yX
-zkybO8wLdbHX9Iw/wc1j50Uf1Z5gHdLf7/hQJoWKpz1RnkWRy6CYON8v1tpVp0tb
-OAajR/kZnzebq2mfa7pyy5zDCX++2kp/dcFwHf31AoGAE8oupBVTZLWj7TBFuP8q
-LkS40U92Sv9v09iDCQVmylmFvUxcXPM2m+7f/qMTNgWrucxzC7kB/6MMWVszHbrz
-vrnCTibnemgx9sZTjKOSxHFOIEw7i85fSa3Cu0qOIDPSnmlwfZpfcMKQrhjLAYhf
-uhooFiLX1X78iZ2OXup4PHUCgYEAsmBrm83sp1V1gAYBBlnVbXakyNv0pCk/Vz61
-iFXeRt1NzDGxLxGw3kQnED8BaIh5kQcyn8Fud7sdzJMv/LAqlT4Ww60mzNYTGyjo
-H3jOsqm3ESfRvduWFreeAQBWbiOczGjV1i8D4EbAFfWT+tjXjchwKBf+6Yt5zn/o
-Bw/uEHUCgYAFs+JPOR25oRyBs7ujrMo/OY1z/eXTVVgZxY+tYGe1FJqDeFyR7ytK
-+JBB1MuDwQKGm2wSIXdCzTNoIx2B9zTseiPTwT8G7vqNFhXoIaTBp4P2xIQb45mJ
-7GkTsMBHwpSMOXgX9Weq3v5xOJ2WxVtjENmd6qzxcYCO5lP15O17hA==
------END RSA PRIVATE KEY-----`
-
-func assertContents(t *testing.T, path string, contents string) {
-	data, err := ioutil.ReadFile(path)
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	if !reflect.DeepEqual(data, []byte(contents)) {
-		t.Fatalf("bad. expected:\n\n%s\n\nGot:\n\n%s", contents, string(data))
-	}
-}
-
-func tempDir(t *testing.T) string {
-	dir, err := ioutil.TempDir("", "tf")
-	if err != nil {
-		t.Fatalf("err: %s", err)
-	}
-	if err := os.RemoveAll(dir); err != nil {
-		t.Fatalf("err: %s", err)
-	}
-
-	return dir
-}
-
-func tempTestFile(t *testing.T) string {
-	dir := tempDir(t)
-	return filepath.Join(dir, "foo")
-}

internal/lang/funcs/collection.go

@@ -311,8 +311,8 @@ var LookupFunc = function.New(&function.Spec{
 			return defaultVal.WithMarks(markses...), nil
 		}
 
-		return cty.UnknownVal(cty.DynamicPseudoType).WithMarks(markses...), fmt.Errorf(
+		return cty.UnknownVal(cty.DynamicPseudoType), fmt.Errorf(
-			"lookup failed to find '%s'", lookupKey)
+			"lookup failed to find key %s", redactIfSensitive(lookupKey, keyMarks))
 	},
 })
 

internal/lang/funcs/collection_test.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"testing"
 
+	"github.com/hashicorp/terraform/internal/lang/marks"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -899,6 +900,46 @@ func TestLookup(t *testing.T) {
 	}
 }
 
+func TestLookup_error(t *testing.T) {
+	simpleMap := cty.MapVal(map[string]cty.Value{
+		"foo": cty.StringVal("bar"),
+	})
+
+	tests := map[string]struct {
+		Values  []cty.Value
+		WantErr string
+	}{
+		"failed to find non-sensitive key": {
+			[]cty.Value{
+				simpleMap,
+				cty.StringVal("boop"),
+			},
+			`lookup failed to find key "boop"`,
+		},
+		"failed to find sensitive key": {
+			[]cty.Value{
+				simpleMap,
+				cty.StringVal("boop").Mark(marks.Sensitive),
+			},
+			"lookup failed to find key (sensitive value)",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			_, err := Lookup(test.Values...)
+
+			if err == nil {
+				t.Fatal("succeeded; want error")
+			}
+
+			if err.Error() != test.WantErr {
+				t.Errorf("wrong error\ngot:  %#v\nwant: %#v", err, test.WantErr)
+			}
+		})
+	}
+}
+
 func TestMatchkeys(t *testing.T) {
 	tests := []struct {
 		Keys      cty.Value

internal/lang/funcs/encoding.go

@@ -20,20 +20,22 @@ var Base64DecodeFunc = function.New(&function.Spec{
 		{
 			Name:        "str",
 			Type:        cty.String,
+			AllowMarked: true,
 		},
 	},
 	Type: function.StaticReturnType(cty.String),
 	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
-		s := args[0].AsString()
+		str, strMarks := args[0].Unmark()
+		s := str.AsString()
 		sDec, err := base64.StdEncoding.DecodeString(s)
 		if err != nil {
-			return cty.UnknownVal(cty.String), fmt.Errorf("failed to decode base64 data '%s'", s)
+			return cty.UnknownVal(cty.String), fmt.Errorf("failed to decode base64 data %s", redactIfSensitive(s, strMarks))
 		}
 		if !utf8.Valid([]byte(sDec)) {
-			log.Printf("[DEBUG] the result of decoding the provided string is not valid UTF-8: %s", sDec)
+			log.Printf("[DEBUG] the result of decoding the provided string is not valid UTF-8: %s", redactIfSensitive(sDec, strMarks))
 			return cty.UnknownVal(cty.String), fmt.Errorf("the result of decoding the provided string is not valid UTF-8")
 		}
-		return cty.StringVal(string(sDec)), nil
+		return cty.StringVal(string(sDec)).WithMarks(strMarks), nil
 	},
 })
 
@@ -125,7 +127,7 @@ var TextDecodeBase64Func = function.New(&function.Spec{
 			case base64.CorruptInputError:
 				return cty.UnknownVal(cty.String), function.NewArgErrorf(0, "the given value is has an invalid base64 symbol at offset %d", int(err))
 			default:
-				return cty.UnknownVal(cty.String), function.NewArgErrorf(0, "invalid source string: %T", err)
+				return cty.UnknownVal(cty.String), function.NewArgErrorf(0, "invalid source string: %w", err)
 			}
 
 		}
@@ -156,13 +158,13 @@ var Base64GzipFunc = function.New(&function.Spec{
 		var b bytes.Buffer
 		gz := gzip.NewWriter(&b)
 		if _, err := gz.Write([]byte(s)); err != nil {
-			return cty.UnknownVal(cty.String), fmt.Errorf("failed to write gzip raw data: '%s'", s)
+			return cty.UnknownVal(cty.String), fmt.Errorf("failed to write gzip raw data: %w", err)
 		}
 		if err := gz.Flush(); err != nil {
-			return cty.UnknownVal(cty.String), fmt.Errorf("failed to flush gzip writer: '%s'", s)
+			return cty.UnknownVal(cty.String), fmt.Errorf("failed to flush gzip writer: %w", err)
 		}
 		if err := gz.Close(); err != nil {
-			return cty.UnknownVal(cty.String), fmt.Errorf("failed to close gzip writer: '%s'", s)
+			return cty.UnknownVal(cty.String), fmt.Errorf("failed to close gzip writer: %w", err)
 		}
 		return cty.StringVal(base64.StdEncoding.EncodeToString(b.Bytes())), nil
 	},

internal/lang/funcs/encoding_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/hashicorp/terraform/internal/lang/marks"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -18,6 +19,11 @@ func TestBase64Decode(t *testing.T) {
 			cty.StringVal("abc123!?$*&()'-=@~"),
 			false,
 		},
+		{
+			cty.StringVal("YWJjMTIzIT8kKiYoKSctPUB+").Mark(marks.Sensitive),
+			cty.StringVal("abc123!?$*&()'-=@~").Mark(marks.Sensitive),
+			false,
+		},
 		{ // Invalid base64 data decoding
 			cty.StringVal("this-is-an-invalid-base64-data"),
 			cty.UnknownVal(cty.String),
@@ -50,6 +56,40 @@ func TestBase64Decode(t *testing.T) {
 	}
 }
 
+func TestBase64Decode_error(t *testing.T) {
+	tests := map[string]struct {
+		String  cty.Value
+		WantErr string
+	}{
+		"invalid base64": {
+			cty.StringVal("dfg"),
+			`failed to decode base64 data "dfg"`,
+		},
+		"sensitive invalid base64": {
+			cty.StringVal("dfg").Mark(marks.Sensitive),
+			`failed to decode base64 data (sensitive value)`,
+		},
+		"invalid utf-8": {
+			cty.StringVal("whee"),
+			"the result of decoding the provided string is not valid UTF-8",
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			_, err := Base64Decode(test.String)
+
+			if err == nil {
+				t.Fatal("succeeded; want error")
+			}
+
+			if err.Error() != test.WantErr {
+				t.Errorf("wrong error result\ngot:  %#v\nwant: %#v", err.Error(), test.WantErr)
+			}
+		})
+	}
+}
+
 func TestBase64Encode(t *testing.T) {
 	tests := []struct {
 		String cty.Value

internal/lang/funcs/filesystem.go

@@ -25,12 +25,14 @@ func MakeFileFunc(baseDir string, encBase64 bool) function.Function {
 			{
 				Name:        "path",
 				Type:        cty.String,
+				AllowMarked: true,
 			},
 		},
 		Type: function.StaticReturnType(cty.String),
 		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
-			path := args[0].AsString()
+			pathArg, pathMarks := args[0].Unmark()
-			src, err := readFileBytes(baseDir, path)
+			path := pathArg.AsString()
+			src, err := readFileBytes(baseDir, path, pathMarks)
 			if err != nil {
 				err = function.NewArgError(0, err)
 				return cty.UnknownVal(cty.String), err
@@ -39,12 +41,12 @@ func MakeFileFunc(baseDir string, encBase64 bool) function.Function {
 			switch {
 			case encBase64:
 				enc := base64.StdEncoding.EncodeToString(src)
-				return cty.StringVal(enc), nil
+				return cty.StringVal(enc).WithMarks(pathMarks), nil
 			default:
 				if !utf8.Valid(src) {
-					return cty.UnknownVal(cty.String), fmt.Errorf("contents of %s are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead", path)
+					return cty.UnknownVal(cty.String), fmt.Errorf("contents of %s are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead", redactIfSensitive(path, pathMarks))
 				}
-				return cty.StringVal(string(src)), nil
+				return cty.StringVal(string(src)).WithMarks(pathMarks), nil
 			}
 		},
 	})
@@ -69,6 +71,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
 		{
 			Name:        "path",
 			Type:        cty.String,
+			AllowMarked: true,
 		},
 		{
 			Name: "vars",
@@ -76,10 +79,10 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
 		},
 	}
 
-	loadTmpl := func(fn string) (hcl.Expression, error) {
+	loadTmpl := func(fn string, marks cty.ValueMarks) (hcl.Expression, error) {
 		// We re-use File here to ensure the same filename interpretation
 		// as it does, along with its other safety checks.
-		tmplVal, err := File(baseDir, cty.StringVal(fn))
+		tmplVal, err := File(baseDir, cty.StringVal(fn).WithMarks(marks))
 		if err != nil {
 			return nil, err
 		}
@@ -159,7 +162,9 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
 			// We'll render our template now to see what result type it produces.
 			// A template consisting only of a single interpolation an potentially
 			// return any type.
-			expr, err := loadTmpl(args[0].AsString())
+
+			pathArg, pathMarks := args[0].Unmark()
+			expr, err := loadTmpl(pathArg.AsString(), pathMarks)
 			if err != nil {
 				return cty.DynamicPseudoType, err
 			}
@@ -170,11 +175,13 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
 			return val.Type(), err
 		},
 		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
-			expr, err := loadTmpl(args[0].AsString())
+			pathArg, pathMarks := args[0].Unmark()
+			expr, err := loadTmpl(pathArg.AsString(), pathMarks)
 			if err != nil {
 				return cty.DynamicVal, err
 			}
-			return renderTmpl(expr, args[1])
+			result, err := renderTmpl(expr, args[1])
+			return result.WithMarks(pathMarks), err
 		},
 	})
 
@@ -188,14 +195,16 @@ func MakeFileExistsFunc(baseDir string) function.Function {
 			{
 				Name:        "path",
 				Type:        cty.String,
+				AllowMarked: true,
 			},
 		},
 		Type: function.StaticReturnType(cty.Bool),
 		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
-			path := args[0].AsString()
+			pathArg, pathMarks := args[0].Unmark()
+			path := pathArg.AsString()
 			path, err := homedir.Expand(path)
 			if err != nil {
-				return cty.UnknownVal(cty.Bool), fmt.Errorf("failed to expand ~: %s", err)
+				return cty.UnknownVal(cty.Bool), fmt.Errorf("failed to expand ~: %w", err)
 			}
 
 			if !filepath.IsAbs(path) {
@@ -208,17 +217,39 @@ func MakeFileExistsFunc(baseDir string) function.Function {
 			fi, err := os.Stat(path)
 			if err != nil {
 				if os.IsNotExist(err) {
-					return cty.False, nil
+					return cty.False.WithMarks(pathMarks), nil
 				}
-				return cty.UnknownVal(cty.Bool), fmt.Errorf("failed to stat %s", path)
+				return cty.UnknownVal(cty.Bool), fmt.Errorf("failed to stat %s", redactIfSensitive(path, pathMarks))
 			}
 
 			if fi.Mode().IsRegular() {
-				return cty.True, nil
+				return cty.True.WithMarks(pathMarks), nil
 			}
 
-			return cty.False, fmt.Errorf("%s is not a regular file, but %q",
+			// The Go stat API only provides convenient access to whether it's
-				path, fi.Mode().String())
+			// a directory or not, so we need to do some bit fiddling to
+			// recognize other irregular file types.
+			filename := redactIfSensitive(path, pathMarks)
+			fileType := fi.Mode().Type()
+			switch {
+			case (fileType & os.ModeDir) != 0:
+				err = function.NewArgErrorf(1, "%s is a directory, not a file", filename)
+			case (fileType & os.ModeDevice) != 0:
+				err = function.NewArgErrorf(1, "%s is a device node, not a regular file", filename)
+			case (fileType & os.ModeNamedPipe) != 0:
+				err = function.NewArgErrorf(1, "%s is a named pipe, not a regular file", filename)
+			case (fileType & os.ModeSocket) != 0:
+				err = function.NewArgErrorf(1, "%s is a unix domain socket, not a regular file", filename)
+			default:
+				// If it's not a type we recognize then we'll just return a
+				// generic error message. This should be very rare.
+				err = function.NewArgErrorf(1, "%s is not a regular file", filename)
+
+				// Note: os.ModeSymlink should be impossible because we used
+				// os.Stat above, not os.Lstat.
+			}
+
+			return cty.False, err
 		},
 	})
 }
@@ -231,16 +262,22 @@ func MakeFileSetFunc(baseDir string) function.Function {
 			{
 				Name:        "path",
 				Type:        cty.String,
+				AllowMarked: true,
 			},
 			{
 				Name:        "pattern",
 				Type:        cty.String,
+				AllowMarked: true,
 			},
 		},
 		Type: function.StaticReturnType(cty.Set(cty.String)),
 		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
-			path := args[0].AsString()
+			pathArg, pathMarks := args[0].Unmark()
-			pattern := args[1].AsString()
+			path := pathArg.AsString()
+			patternArg, patternMarks := args[1].Unmark()
+			pattern := patternArg.AsString()
+
+			marks := []cty.ValueMarks{pathMarks, patternMarks}
 
 			if !filepath.IsAbs(path) {
 				path = filepath.Join(baseDir, path)
@@ -253,7 +290,7 @@ func MakeFileSetFunc(baseDir string) function.Function {
 
 			matches, err := doublestar.Glob(pattern)
 			if err != nil {
-				return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to glob pattern (%s): %s", pattern, err)
+				return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to glob pattern %s: %w", redactIfSensitive(pattern, marks...), err)
 			}
 
 			var matchVals []cty.Value
@@ -261,7 +298,7 @@ func MakeFileSetFunc(baseDir string) function.Function {
 				fi, err := os.Stat(match)
 
 				if err != nil {
-					return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to stat (%s): %s", match, err)
+					return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to stat %s: %w", redactIfSensitive(match, marks...), err)
 				}
 
 				if !fi.Mode().IsRegular() {
@@ -272,7 +309,7 @@ func MakeFileSetFunc(baseDir string) function.Function {
 				match, err = filepath.Rel(path, match)
 
 				if err != nil {
-					return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to trim path of match (%s): %s", match, err)
+					return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to trim path of match %s: %w", redactIfSensitive(match, marks...), err)
 				}
 
 				// Replace any remaining file separators with forward slash (/)
@@ -283,10 +320,10 @@ func MakeFileSetFunc(baseDir string) function.Function {
 			}
 
 			if len(matchVals) == 0 {
-				return cty.SetValEmpty(cty.String), nil
+				return cty.SetValEmpty(cty.String).WithMarks(marks...), nil
 			}
 
-			return cty.SetVal(matchVals), nil
+			return cty.SetVal(matchVals).WithMarks(marks...), nil
 		},
 	})
 }
@@ -355,7 +392,7 @@ var PathExpandFunc = function.New(&function.Spec{
 func openFile(baseDir, path string) (*os.File, error) {
 	path, err := homedir.Expand(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to expand ~: %s", err)
+		return nil, fmt.Errorf("failed to expand ~: %w", err)
 	}
 
 	if !filepath.IsAbs(path) {
@@ -368,12 +405,12 @@ func openFile(baseDir, path string) (*os.File, error) {
 	return os.Open(path)
 }
 
-func readFileBytes(baseDir, path string) ([]byte, error) {
+func readFileBytes(baseDir, path string, marks cty.ValueMarks) ([]byte, error) {
 	f, err := openFile(baseDir, path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// An extra Terraform-specific hint for this situation
-			return nil, fmt.Errorf("no file exists at %s; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource", path)
+			return nil, fmt.Errorf("no file exists at %s; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource", redactIfSensitive(path, marks))
 		}
 		return nil, err
 	}
@@ -381,7 +418,7 @@ func readFileBytes(baseDir, path string) ([]byte, error) {
 
 	src, err := ioutil.ReadAll(f)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read %s", path)
+		return nil, fmt.Errorf("failed to read file: %w", err)
 	}
 
 	return src, nil

internal/lang/funcs/filesystem_test.go

@@ -2,9 +2,11 @@ package funcs
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/hashicorp/terraform/internal/lang/marks"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
@@ -15,22 +17,32 @@ func TestFile(t *testing.T) {
 	tests := []struct {
 		Path cty.Value
 		Want cty.Value
-		Err  bool
+		Err  string
 	}{
 		{
 			cty.StringVal("testdata/hello.txt"),
 			cty.StringVal("Hello World"),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata/icon.png"),
 			cty.NilVal,
-			true, // Not valid UTF-8
+			`contents of "testdata/icon.png" are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
+		},
+		{
+			cty.StringVal("testdata/icon.png").Mark(marks.Sensitive),
+			cty.NilVal,
+			`contents of (sensitive value) are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
 		},
 		{
 			cty.StringVal("testdata/missing"),
 			cty.NilVal,
-			true, // no file exists
+			`no file exists at "testdata/missing"; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource`,
+		},
+		{
+			cty.StringVal("testdata/missing").Mark(marks.Sensitive),
+			cty.NilVal,
+			`no file exists at (sensitive value); this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource`,
 		},
 	}
 
@@ -38,10 +50,13 @@ func TestFile(t *testing.T) {
 		t.Run(fmt.Sprintf("File(\".\", %#v)", test.Path), func(t *testing.T) {
 			got, err := File(".", test.Path)
 
-			if test.Err {
+			if test.Err != "" {
 				if err == nil {
 					t.Fatal("succeeded; want error")
 				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
 				return
 			} else if err != nil {
 				t.Fatalf("unexpected error: %s", err)
@@ -71,13 +86,19 @@ func TestTemplateFile(t *testing.T) {
 			cty.StringVal("testdata/icon.png"),
 			cty.EmptyObjectVal,
 			cty.NilVal,
-			`contents of testdata/icon.png are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
+			`contents of "testdata/icon.png" are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
 		},
 		{
 			cty.StringVal("testdata/missing"),
 			cty.EmptyObjectVal,
 			cty.NilVal,
-			`no file exists at testdata/missing; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource`,
+			`no file exists at "testdata/missing"; this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource`,
+		},
+		{
+			cty.StringVal("testdata/secrets.txt").Mark(marks.Sensitive),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`no file exists at (sensitive value); this function works only with files that are distributed as part of the configuration source code, so if this file will be created by a resource in this configuration you must instead obtain this result from an attribute of that resource`,
 		},
 		{
 			cty.StringVal("testdata/hello.tmpl"),
@@ -197,33 +218,61 @@ func TestFileExists(t *testing.T) {
 	tests := []struct {
 		Path cty.Value
 		Want cty.Value
-		Err  bool
+		Err  string
 	}{
 		{
 			cty.StringVal("testdata/hello.txt"),
 			cty.BoolVal(true),
-			false,
+			``,
 		},
 		{
-			cty.StringVal(""), // empty path
+			cty.StringVal(""),
 			cty.BoolVal(false),
-			true,
+			`"." is a directory, not a file`,
+		},
+		{
+			cty.StringVal("testdata").Mark(marks.Sensitive),
+			cty.BoolVal(false),
+			`(sensitive value) is a directory, not a file`,
 		},
 		{
 			cty.StringVal("testdata/missing"),
 			cty.BoolVal(false),
-			false, // no file exists
+			``,
+		},
+		{
+			cty.StringVal("testdata/unreadable/foobar"),
+			cty.BoolVal(false),
+			`failed to stat "testdata/unreadable/foobar"`,
+		},
+		{
+			cty.StringVal("testdata/unreadable/foobar").Mark(marks.Sensitive),
+			cty.BoolVal(false),
+			`failed to stat (sensitive value)`,
 		},
 	}
 
+	// Ensure "unreadable" directory cannot be listed during the test run
+	fi, err := os.Lstat("testdata/unreadable")
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Chmod("testdata/unreadable", 0000)
+	defer func(mode os.FileMode) {
+		os.Chmod("testdata/unreadable", mode)
+	}(fi.Mode())
+
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("FileExists(\".\", %#v)", test.Path), func(t *testing.T) {
 			got, err := FileExists(".", test.Path)
 
-			if test.Err {
+			if test.Err != "" {
 				if err == nil {
 					t.Fatal("succeeded; want error")
 				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
 				return
 			} else if err != nil {
 				t.Fatalf("unexpected error: %s", err)
@@ -241,49 +290,49 @@ func TestFileSet(t *testing.T) {
 		Path    cty.Value
 		Pattern cty.Value
 		Want    cty.Value
-		Err     bool
+		Err     string
 	}{
 		{
 			cty.StringVal("."),
 			cty.StringVal("testdata*"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("testdata"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("{testdata,missing}"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("testdata/missing"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("testdata/missing*"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("*/missing"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("**/missing"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -291,7 +340,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -299,7 +348,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -307,7 +356,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -316,7 +365,7 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("testdata/hello.tmpl"),
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -325,7 +374,7 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("testdata/hello.tmpl"),
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -333,7 +382,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -341,7 +390,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -350,7 +399,7 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("testdata/hello.tmpl"),
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -359,7 +408,7 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("testdata/hello.tmpl"),
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
@@ -368,31 +417,37 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("testdata/hello.tmpl"),
 				cty.StringVal("testdata/hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("["),
 			cty.SetValEmpty(cty.String),
-			true,
+			`failed to glob pattern "[": syntax error in pattern`,
+		},
+		{
+			cty.StringVal("."),
+			cty.StringVal("[").Mark(marks.Sensitive),
+			cty.SetValEmpty(cty.String),
+			`failed to glob pattern (sensitive value): syntax error in pattern`,
 		},
 		{
 			cty.StringVal("."),
 			cty.StringVal("\\"),
 			cty.SetValEmpty(cty.String),
-			true,
+			`failed to glob pattern "\\": syntax error in pattern`,
 		},
 		{
 			cty.StringVal("testdata"),
 			cty.StringVal("missing"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata"),
 			cty.StringVal("missing*"),
 			cty.SetValEmpty(cty.String),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata"),
@@ -400,7 +455,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata"),
@@ -408,7 +463,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata"),
@@ -416,7 +471,7 @@ func TestFileSet(t *testing.T) {
 			cty.SetVal([]cty.Value{
 				cty.StringVal("hello.txt"),
 			}),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("testdata"),
@@ -425,7 +480,7 @@ func TestFileSet(t *testing.T) {
 				cty.StringVal("hello.tmpl"),
 				cty.StringVal("hello.txt"),
 			}),
-			false,
+			``,
 		},
 	}
 
@@ -433,10 +488,13 @@ func TestFileSet(t *testing.T) {
 		t.Run(fmt.Sprintf("FileSet(\".\", %#v, %#v)", test.Path, test.Pattern), func(t *testing.T) {
 			got, err := FileSet(".", test.Path, test.Pattern)
 
-			if test.Err {
+			if test.Err != "" {
 				if err == nil {
 					t.Fatal("succeeded; want error")
 				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
 				return
 			} else if err != nil {
 				t.Fatalf("unexpected error: %s", err)

internal/lang/funcs/number.go

@@ -97,10 +97,12 @@ var ParseIntFunc = function.New(&function.Spec{
 		{
 			Name:        "number",
 			Type:        cty.DynamicPseudoType,
+			AllowMarked: true,
 		},
 		{
 			Name:        "base",
 			Type:        cty.Number,
+			AllowMarked: true,
 		},
 	},
 
@@ -116,11 +118,13 @@ var ParseIntFunc = function.New(&function.Spec{
 		var base int
 		var err error
 
-		if err = gocty.FromCtyValue(args[0], &numstr); err != nil {
+		numArg, numMarks := args[0].Unmark()
+		if err = gocty.FromCtyValue(numArg, &numstr); err != nil {
 			return cty.UnknownVal(cty.String), function.NewArgError(0, err)
 		}
 
-		if err = gocty.FromCtyValue(args[1], &base); err != nil {
+		baseArg, baseMarks := args[1].Unmark()
+		if err = gocty.FromCtyValue(baseArg, &base); err != nil {
 			return cty.UnknownVal(cty.Number), function.NewArgError(1, err)
 		}
 
@@ -135,13 +139,13 @@ var ParseIntFunc = function.New(&function.Spec{
 		if !ok {
 			return cty.UnknownVal(cty.Number), function.NewArgErrorf(
 				0,
-				"cannot parse %q as a base %d integer",
+				"cannot parse %s as a base %s integer",
-				numstr,
+				redactIfSensitive(numstr, numMarks),
-				base,
+				redactIfSensitive(base, baseMarks),
 			)
 		}
 
-		parsedNum := cty.NumberVal((&big.Float{}).SetInt(num))
+		parsedNum := cty.NumberVal((&big.Float{}).SetInt(num)).WithMarks(numMarks, baseMarks)
 
 		return parsedNum, nil
 	},

internal/lang/funcs/number_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/hashicorp/terraform/internal/lang/marks"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -187,139 +188,175 @@ func TestParseInt(t *testing.T) {
 		Num  cty.Value
 		Base cty.Value
 		Want cty.Value
-		Err  bool
+		Err  string
 	}{
 		{
 			cty.StringVal("128"),
 			cty.NumberIntVal(10),
 			cty.NumberIntVal(128),
-			false,
+			``,
+		},
+		{
+			cty.StringVal("128").Mark(marks.Sensitive),
+			cty.NumberIntVal(10),
+			cty.NumberIntVal(128).Mark(marks.Sensitive),
+			``,
+		},
+		{
+			cty.StringVal("128"),
+			cty.NumberIntVal(10).Mark(marks.Sensitive),
+			cty.NumberIntVal(128).Mark(marks.Sensitive),
+			``,
+		},
+		{
+			cty.StringVal("128").Mark(marks.Sensitive),
+			cty.NumberIntVal(10).Mark(marks.Sensitive),
+			cty.NumberIntVal(128).Mark(marks.Sensitive),
+			``,
+		},
+		{
+			cty.StringVal("128").Mark(marks.Raw),
+			cty.NumberIntVal(10).Mark(marks.Sensitive),
+			cty.NumberIntVal(128).WithMarks(cty.NewValueMarks(marks.Raw, marks.Sensitive)),
+			``,
 		},
 		{
 			cty.StringVal("-128"),
 			cty.NumberIntVal(10),
 			cty.NumberIntVal(-128),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("00128"),
 			cty.NumberIntVal(10),
 			cty.NumberIntVal(128),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("-00128"),
 			cty.NumberIntVal(10),
 			cty.NumberIntVal(-128),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("FF00"),
 			cty.NumberIntVal(16),
 			cty.NumberIntVal(65280),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("ff00"),
 			cty.NumberIntVal(16),
 			cty.NumberIntVal(65280),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("-FF00"),
 			cty.NumberIntVal(16),
 			cty.NumberIntVal(-65280),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("00FF00"),
 			cty.NumberIntVal(16),
 			cty.NumberIntVal(65280),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("-00FF00"),
 			cty.NumberIntVal(16),
 			cty.NumberIntVal(-65280),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("1011111011101111"),
 			cty.NumberIntVal(2),
 			cty.NumberIntVal(48879),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("aA"),
 			cty.NumberIntVal(62),
 			cty.NumberIntVal(656),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("Aa"),
 			cty.NumberIntVal(62),
 			cty.NumberIntVal(2242),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("999999999999999999999999999999999999999999999999999999999999"),
 			cty.NumberIntVal(10),
 			cty.MustParseNumberVal("999999999999999999999999999999999999999999999999999999999999"),
-			false,
+			``,
 		},
 		{
 			cty.StringVal("FF"),
 			cty.NumberIntVal(10),
 			cty.UnknownVal(cty.Number),
-			true,
+			`cannot parse "FF" as a base 10 integer`,
+		},
+		{
+			cty.StringVal("FF").Mark(marks.Sensitive),
+			cty.NumberIntVal(10),
+			cty.UnknownVal(cty.Number),
+			`cannot parse (sensitive value) as a base 10 integer`,
+		},
+		{
+			cty.StringVal("FF").Mark(marks.Sensitive),
+			cty.NumberIntVal(10).Mark(marks.Sensitive),
+			cty.UnknownVal(cty.Number),
+			`cannot parse (sensitive value) as a base (sensitive value) integer`,
 		},
 		{
 			cty.StringVal("00FF"),
 			cty.NumberIntVal(10),
 			cty.UnknownVal(cty.Number),
-			true,
+			`cannot parse "00FF" as a base 10 integer`,
 		},
 		{
 			cty.StringVal("-00FF"),
 			cty.NumberIntVal(10),
 			cty.UnknownVal(cty.Number),
-			true,
+			`cannot parse "-00FF" as a base 10 integer`,
 		},
 		{
 			cty.NumberIntVal(2),
 			cty.NumberIntVal(10),
 			cty.UnknownVal(cty.Number),
-			true,
+			`first argument must be a string, not number`,
 		},
 		{
 			cty.StringVal("1"),
 			cty.NumberIntVal(63),
 			cty.UnknownVal(cty.Number),
-			true,
+			`base must be a whole number between 2 and 62 inclusive`,
 		},
 		{
 			cty.StringVal("1"),
 			cty.NumberIntVal(-1),
 			cty.UnknownVal(cty.Number),
-			true,
+			`base must be a whole number between 2 and 62 inclusive`,
 		},
 		{
 			cty.StringVal("1"),
 			cty.NumberIntVal(1),
 			cty.UnknownVal(cty.Number),
-			true,
+			`base must be a whole number between 2 and 62 inclusive`,
 		},
 		{
 			cty.StringVal("1"),
 			cty.NumberIntVal(0),
 			cty.UnknownVal(cty.Number),
-			true,
+			`base must be a whole number between 2 and 62 inclusive`,
 		},
 		{
 			cty.StringVal("1.2"),
 			cty.NumberIntVal(10),
 			cty.UnknownVal(cty.Number),
-			true,
+			`cannot parse "1.2" as a base 10 integer`,
 		},
 	}
 
@@ -327,10 +364,13 @@ func TestParseInt(t *testing.T) {
 		t.Run(fmt.Sprintf("parseint(%#v, %#v)", test.Num, test.Base), func(t *testing.T) {
 			got, err := ParseInt(test.Num, test.Base)
 
-			if test.Err {
+			if test.Err != "" {
 				if err == nil {
 					t.Fatal("succeeded; want error")
 				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
 				return
 			} else if err != nil {
 				t.Fatalf("unexpected error: %s", err)

internal/lang/funcs/redact.go

@@ -0,0 +1,20 @@
+package funcs
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform/internal/lang/marks"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func redactIfSensitive(value interface{}, markses ...cty.ValueMarks) string {
+	if marks.Has(cty.DynamicVal.WithMarks(markses...), marks.Sensitive) {
+		return "(sensitive value)"
+	}
+	switch v := value.(type) {
+	case string:
+		return fmt.Sprintf("%q", v)
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}

internal/lang/funcs/redact_test.go

@@ -0,0 +1,51 @@
+package funcs
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/internal/lang/marks"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestRedactIfSensitive(t *testing.T) {
+	testCases := map[string]struct {
+		value interface{}
+		marks []cty.ValueMarks
+		want  string
+	}{
+		"sensitive string": {
+			value: "foo",
+			marks: []cty.ValueMarks{cty.NewValueMarks(marks.Sensitive)},
+			want:  "(sensitive value)",
+		},
+		"raw non-sensitive string": {
+			value: "foo",
+			marks: []cty.ValueMarks{cty.NewValueMarks(marks.Raw)},
+			want:  `"foo"`,
+		},
+		"raw sensitive string": {
+			value: "foo",
+			marks: []cty.ValueMarks{cty.NewValueMarks(marks.Raw), cty.NewValueMarks(marks.Sensitive)},
+			want:  "(sensitive value)",
+		},
+		"sensitive number": {
+			value: 12345,
+			marks: []cty.ValueMarks{cty.NewValueMarks(marks.Sensitive)},
+			want:  "(sensitive value)",
+		},
+		"non-sensitive number": {
+			value: 12345,
+			marks: []cty.ValueMarks{},
+			want:  "12345",
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			got := redactIfSensitive(tc.value, tc.marks...)
+			if got != tc.want {
+				t.Errorf("wrong result, got %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

internal/plugin6/grpc_provider.go

@@ -436,6 +436,8 @@ func (p *GRPCProvider) PlanResourceChange(r providers.PlanResourceChangeRequest)
 
 	resp.PlannedPrivate = protoResp.PlannedPrivate
 
+	resp.LegacyTypeSystem = protoResp.LegacyTypeSystem
+
 	return resp
 }
 
@@ -494,6 +496,8 @@ func (p *GRPCProvider) ApplyResourceChange(r providers.ApplyResourceChangeReques
 	}
 	resp.NewState = state
 
+	resp.LegacyTypeSystem = protoResp.LegacyTypeSystem
+
 	return resp
 }
 

internal/refactoring/move_execute.go

@@ -31,6 +31,10 @@ func ApplyMoves(stmts []MoveStatement, state *states.State) MoveResults {
 		Blocked: make(map[addrs.UniqueKey]MoveBlocked),
 	}
 
+	if len(stmts) == 0 {
+		return ret
+	}
+
 	// The methodology here is to construct a small graph of all of the move
 	// statements where the edges represent where a particular statement
 	// is either chained from or nested inside the effect of another statement.
@@ -39,13 +43,18 @@ func ApplyMoves(stmts []MoveStatement, state *states.State) MoveResults {
 
 	g := buildMoveStatementGraph(stmts)
 
-	// If there are any cycles in the graph then we'll not take any action
+	// If the graph is not valid the we will not take any action at all. The
-	// at all. The separate validation step should detect this and return
+	// separate validation step should detect this and return an error.
-	// an error.
+	if diags := validateMoveStatementGraph(g); diags.HasErrors() {
-	if len(g.Cycles()) != 0 {
+		log.Printf("[ERROR] ApplyMoves: %s", diags.ErrWithWarnings())
 		return ret
 	}
 
+	// The graph must be reduced in order for ReverseDepthFirstWalk to work
+	// correctly, since it is built from following edges and can skip over
+	// dependencies if there is a direct edge to a transitive dependency.
+	g.TransitiveReduction()
+
 	// The starting nodes are the ones that don't depend on any other nodes.
 	startNodes := make(dag.Set, len(stmts))
 	for _, v := range g.Vertices() {
@@ -88,16 +97,13 @@ func ApplyMoves(stmts []MoveStatement, state *states.State) MoveResults {
 
 		for _, ms := range state.Modules {
 			modAddr := ms.Addr
-			if !stmt.From.SelectsModule(modAddr) {
-				continue
-			}
 
-			// We now know that the current module is relevant but what
+			// We don't yet know that the current module is relevant, and
-			// we'll do with it depends on the object kind.
+			// we determine that differently for each the object kind.
 			switch kind := stmt.ObjectKind(); kind {
 			case addrs.MoveEndpointModule:
 				// For a module endpoint we just try the module address
-				// directly.
+				// directly, and execute the moves if it matches.
 				if newAddr, matches := modAddr.MoveDestination(stmt.From, stmt.To); matches {
 					log.Printf("[TRACE] refactoring.ApplyMoves: %s has moved to %s", modAddr, newAddr)
 
@@ -125,8 +131,15 @@ func ApplyMoves(stmts []MoveStatement, state *states.State) MoveResults {
 					continue
 				}
 			case addrs.MoveEndpointResource:
-				// For a resource endpoint we need to search each of the
+				// For a resource endpoint we require an exact containing
-				// resources and resource instances in the module.
+				// module match, because by definition a matching resource
+				// cannot be nested any deeper than that.
+				if !stmt.From.SelectsModule(modAddr) {
+					continue
+				}
+
+				// We then need to search each of the resources and resource
+				// instances in the module.
 				for _, rs := range ms.Resources {
 					rAddr := rs.Addr
 					if newAddr, matches := rAddr.MoveDestination(stmt.From, stmt.To); matches {
@@ -238,11 +251,31 @@ func statementDependsOn(a, b *MoveStatement) bool {
 	//
 	// Since we are only interested in checking if A depends on B, we only need
 	// to check the 4 possibilities above which result in B being executed
-	// first.
+	// first. If we're there's no dependency at all we can return immediately.
-	return a.From.NestedWithin(b.To) ||
+	if !(a.From.NestedWithin(b.To) || a.To.NestedWithin(b.To) ||
-		a.To.NestedWithin(b.To) ||
+		b.From.NestedWithin(a.From) || b.To.NestedWithin(a.From)) {
-		b.From.NestedWithin(a.From) ||
+		return false
-		b.To.NestedWithin(a.From)
+	}
+
+	// If a nested move has a dependency, we need to rule out the possibility
+	// that this is a move inside a module only changing indexes. If an
+	// ancestor module is only changing the index of a nested module, any
+	// nested move statements are going to match both the From and To address
+	// when the base name is not changing, causing a cycle in the order of
+	// operations.
+
+	// if A is not declared in an ancestor module, then we can't be nested
+	// within a module index change.
+	if len(a.To.Module()) >= len(b.To.Module()) {
+		return true
+	}
+	// We only want the nested move statement to depend on the outer module
+	// move, so we only test this in the reverse direction.
+	if a.From.IsModuleReIndex(a.To) {
+		return false
+	}
+
+	return true
 }
 
 // MoveResults describes the outcome of an ApplyMoves call.

internal/refactoring/move_execute_test.go

@@ -21,7 +21,10 @@ func TestApplyMoves(t *testing.T) {
 	}
 
 	moduleBoo, _ := addrs.ParseModuleInstanceStr("module.boo")
+	moduleBar, _ := addrs.ParseModuleInstanceStr("module.bar")
 	moduleBarKey, _ := addrs.ParseModuleInstanceStr("module.bar[0]")
+	moduleBooHoo, _ := addrs.ParseModuleInstanceStr("module.boo.module.hoo")
+	moduleBarHoo, _ := addrs.ParseModuleInstanceStr("module.bar.module.hoo")
 
 	instAddrs := map[string]addrs.AbsResourceInstance{
 		"foo.from": addrs.Resource{
@@ -84,6 +87,12 @@ func TestApplyMoves(t *testing.T) {
 			Name: "to",
 		}.Instance(addrs.IntKey(0)).Absolute(moduleBoo),
 
+		"module.bar.foo.from": addrs.Resource{
+			Mode: addrs.ManagedResourceMode,
+			Type: "foo",
+			Name: "from",
+		}.Instance(addrs.NoKey).Absolute(moduleBar),
+
 		"module.bar[0].foo.from": addrs.Resource{
 			Mode: addrs.ManagedResourceMode,
 			Type: "foo",
@@ -113,6 +122,18 @@ func TestApplyMoves(t *testing.T) {
 			Type: "foo",
 			Name: "to",
 		}.Instance(addrs.IntKey(0)).Absolute(moduleBarKey),
+
+		"module.boo.module.hoo.foo.from": addrs.Resource{
+			Mode: addrs.ManagedResourceMode,
+			Type: "foo",
+			Name: "from",
+		}.Instance(addrs.NoKey).Absolute(moduleBooHoo),
+
+		"module.bar.module.hoo.foo.from": addrs.Resource{
+			Mode: addrs.ManagedResourceMode,
+			Type: "foo",
+			Name: "from",
+		}.Instance(addrs.NoKey).Absolute(moduleBarHoo),
 	}
 
 	emptyResults := MoveResults{
@@ -289,6 +310,47 @@ func TestApplyMoves(t *testing.T) {
 			},
 		},
 
+		"module move with child module": {
+			[]MoveStatement{
+				testMoveStatement(t, "", "module.boo", "module.bar"),
+			},
+			states.BuildState(func(s *states.SyncState) {
+				s.SetResourceInstanceCurrent(
+					instAddrs["module.boo.foo.from"],
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					providerAddr,
+				)
+				s.SetResourceInstanceCurrent(
+					instAddrs["module.boo.module.hoo.foo.from"],
+					&states.ResourceInstanceObjectSrc{
+						Status:    states.ObjectReady,
+						AttrsJSON: []byte(`{}`),
+					},
+					providerAddr,
+				)
+			}),
+			MoveResults{
+				Changes: map[addrs.UniqueKey]MoveSuccess{
+					instAddrs["module.bar.foo.from"].UniqueKey(): {
+						From: instAddrs["module.boo.foo.from"],
+						To:   instAddrs["module.bar.foo.from"],
+					},
+					instAddrs["module.bar.module.hoo.foo.from"].UniqueKey(): {
+						From: instAddrs["module.boo.module.hoo.foo.from"],
+						To:   instAddrs["module.bar.module.hoo.foo.from"],
+					},
+				},
+				Blocked: map[addrs.UniqueKey]MoveBlocked{},
+			},
+			[]string{
+				`module.bar.foo.from`,
+				`module.bar.module.hoo.foo.from`,
+			},
+		},
+
 		"move whole single module to indexed module": {
 			[]MoveStatement{
 				testMoveStatement(t, "", "module.boo", "module.bar[0]"),

internal/refactoring/move_statement.go

@@ -149,7 +149,7 @@ func impliedMoveStatements(cfg *configs.Config, prevRunState *states.State, expl
 	}
 
 	for _, childCfg := range cfg.Children {
-		into = findMoveStatements(childCfg, into)
+		into = impliedMoveStatements(childCfg, prevRunState, explicitStmts, into)
 	}
 
 	return into

internal/refactoring/move_statement_test.go

@@ -18,6 +18,15 @@ func TestImpliedMoveStatements(t *testing.T) {
 			Name: name,
 		}.Absolute(addrs.RootModuleInstance)
 	}
+
+	nestedResourceAddr := func(mod, name string) addrs.AbsResource {
+		return addrs.Resource{
+			Mode: addrs.ManagedResourceMode,
+			Type: "foo",
+			Name: name,
+		}.Absolute(addrs.RootModuleInstance.Child(mod, addrs.NoKey))
+	}
+
 	instObjState := func() *states.ResourceInstanceObjectSrc {
 		return &states.ResourceInstanceObjectSrc{}
 	}
@@ -86,6 +95,19 @@ func TestImpliedMoveStatements(t *testing.T) {
 			instObjState(),
 			providerAddr,
 		)
+
+		// Add two resource nested in a module to ensure we find these
+		// recursively.
+		s.SetResourceInstanceCurrent(
+			nestedResourceAddr("child", "formerly_count").Instance(addrs.IntKey(0)),
+			instObjState(),
+			providerAddr,
+		)
+		s.SetResourceInstanceCurrent(
+			nestedResourceAddr("child", "now_count").Instance(addrs.NoKey),
+			instObjState(),
+			providerAddr,
+		)
 	})
 
 	explicitStmts := FindMoveStatements(rootCfg)
@@ -101,6 +123,19 @@ func TestImpliedMoveStatements(t *testing.T) {
 				End:      tfdiags.SourcePos{Line: 5, Column: 32, Byte: 211},
 			},
 		},
+
+		// Found implied moves in a nested module, ignoring the explicit moves
+		{
+			From:    addrs.ImpliedMoveStatementEndpoint(nestedResourceAddr("child", "formerly_count").Instance(addrs.IntKey(0)), tfdiags.SourceRange{}),
+			To:      addrs.ImpliedMoveStatementEndpoint(nestedResourceAddr("child", "formerly_count").Instance(addrs.NoKey), tfdiags.SourceRange{}),
+			Implied: true,
+			DeclRange: tfdiags.SourceRange{
+				Filename: "testdata/move-statement-implied/child/move-statement-implied.tf",
+				Start:    tfdiags.SourcePos{Line: 5, Column: 1, Byte: 180},
+				End:      tfdiags.SourcePos{Line: 5, Column: 32, Byte: 211},
+			},
+		},
+
 		{
 			From:    addrs.ImpliedMoveStatementEndpoint(resourceAddr("now_count").Instance(addrs.NoKey), tfdiags.SourceRange{}),
 			To:      addrs.ImpliedMoveStatementEndpoint(resourceAddr("now_count").Instance(addrs.IntKey(0)), tfdiags.SourceRange{}),
@@ -112,6 +147,18 @@ func TestImpliedMoveStatements(t *testing.T) {
 			},
 		},
 
+		// Found implied moves in a nested module, ignoring the explicit moves
+		{
+			From:    addrs.ImpliedMoveStatementEndpoint(nestedResourceAddr("child", "now_count").Instance(addrs.NoKey), tfdiags.SourceRange{}),
+			To:      addrs.ImpliedMoveStatementEndpoint(nestedResourceAddr("child", "now_count").Instance(addrs.IntKey(0)), tfdiags.SourceRange{}),
+			Implied: true,
+			DeclRange: tfdiags.SourceRange{
+				Filename: "testdata/move-statement-implied/child/move-statement-implied.tf",
+				Start:    tfdiags.SourcePos{Line: 10, Column: 11, Byte: 282},
+				End:      tfdiags.SourcePos{Line: 10, Column: 12, Byte: 283},
+			},
+		},
+
 		// We generate foo.ambiguous[0] to foo.ambiguous here, even though
 		// there's already a foo.ambiguous in the state, because it's the
 		// responsibility of the later ApplyMoves step to deal with the

internal/refactoring/move_validate.go

@@ -8,6 +8,7 @@ import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/configs"
+	"github.com/hashicorp/terraform/internal/dag"
 	"github.com/hashicorp/terraform/internal/instances"
 	"github.com/hashicorp/terraform/internal/tfdiags"
 )
@@ -31,6 +32,10 @@ import (
 func ValidateMoves(stmts []MoveStatement, rootCfg *configs.Config, declaredInsts instances.Set) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 
+	if len(stmts) == 0 {
+		return diags
+	}
+
 	g := buildMoveStatementGraph(stmts)
 
 	// We need to track the absolute versions of our endpoint addresses in
@@ -50,6 +55,12 @@ func ValidateMoves(stmts []MoveStatement, rootCfg *configs.Config, declaredInsts
 		_, toCallSteps := stmt.To.ModuleCallTraversals()
 
 		modCfg := rootCfg.Descendent(stmtMod)
+		if !stmt.Implied {
+			// Implied statements can cross module boundaries because we
+			// generate them only for changing instance keys on a single
+			// resource. They happen to be generated _as if_ they were written
+			// in the root module, but the source and destination are always
+			// in the same module anyway.
 			if pkgAddr := callsThroughModulePackage(modCfg, fromCallSteps); pkgAddr != nil {
 				diags = diags.Append(&hcl.Diagnostic{
 					Severity: hcl.DiagError,
@@ -72,6 +83,7 @@ func ValidateMoves(stmts []MoveStatement, rootCfg *configs.Config, declaredInsts
 					Subject: stmt.DeclRange.ToHCL().Ptr(),
 				})
 			}
+		}
 
 		for _, modInst := range declaredInsts.InstancesForModule(stmtMod) {
 
@@ -200,6 +212,14 @@ func ValidateMoves(stmts []MoveStatement, rootCfg *configs.Config, declaredInsts
 	// validation rules above where we can make better suggestions, and so
 	// we'll use a cycle report only as a last resort.
 	if !diags.HasErrors() {
+		diags = diags.Append(validateMoveStatementGraph(g))
+	}
+
+	return diags
+}
+
+func validateMoveStatementGraph(g *dag.AcyclicGraph) tfdiags.Diagnostics {
+	var diags tfdiags.Diagnostics
 	for _, cycle := range g.Cycles() {
 		// Reporting cycles is awkward because there isn't any definitive
 		// way to decide which of the objects in the cycle is the cause of
@@ -227,6 +247,22 @@ func ValidateMoves(stmts []MoveStatement, rootCfg *configs.Config, declaredInsts
 			),
 		))
 	}
+
+	// Look for cycles to self.
+	// A user shouldn't be able to create self-references, but we cannot
+	// correctly process a graph with them.
+	for _, e := range g.Edges() {
+		src := e.Source()
+		if src == e.Target() {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Self reference in move statements",
+				fmt.Sprintf(
+					"The move statement %s refers to itself the move dependency graph, which is invalid. This is a bug in Terraform; please report it!",
+					src.(*MoveStatement).Name(),
+				),
+			))
+		}
 	}
 
 	return diags

internal/refactoring/move_validate_test.go

@@ -366,6 +366,50 @@ Each resource can have moved from only one source resource.`,
 			},
 			WantError: `Cross-package move statement: This statement declares a move to an object declared in external module package "fake-external:///". Move statements can be only within a single module package.`,
 		},
+		"implied move from resource in another module package": {
+			Statements: []MoveStatement{
+				makeTestImpliedMoveStmt(t,
+					``,
+					`module.fake_external.test.thing`,
+					`test.thing`,
+				),
+			},
+			// Implied move statements are not subject to the cross-package restriction
+			WantError: ``,
+		},
+		"implied move to resource in another module package": {
+			Statements: []MoveStatement{
+				makeTestImpliedMoveStmt(t,
+					``,
+					`test.thing`,
+					`module.fake_external.test.thing`,
+				),
+			},
+			// Implied move statements are not subject to the cross-package restriction
+			WantError: ``,
+		},
+		"implied move from module call in another module package": {
+			Statements: []MoveStatement{
+				makeTestImpliedMoveStmt(t,
+					``,
+					`module.fake_external.module.a`,
+					`module.b`,
+				),
+			},
+			// Implied move statements are not subject to the cross-package restriction
+			WantError: ``,
+		},
+		"implied move to module call in another module package": {
+			Statements: []MoveStatement{
+				makeTestImpliedMoveStmt(t,
+					``,
+					`module.a`,
+					`module.fake_external.module.b`,
+				),
+			},
+			// Implied move statements are not subject to the cross-package restriction
+			WantError: ``,
+		},
 		"move to a call that refers to another module package": {
 			Statements: []MoveStatement{
 				makeTestMoveStmt(t,
@@ -404,6 +448,58 @@ Each resource can have moved from only one source resource.`,
 			},
 			WantError: `Resource type mismatch: This statement declares a move from test.nonexist1[0] to other.single, which is a resource instance of a different type.`,
 		},
+		"crossing nested statements": {
+			// overlapping nested moves will result in a cycle.
+			Statements: []MoveStatement{
+				makeTestMoveStmt(t, ``,
+					`module.nonexist.test.single`,
+					`module.count[0].test.count[0]`,
+				),
+				makeTestMoveStmt(t, ``,
+					`module.nonexist`,
+					`module.count[0]`,
+				),
+			},
+			WantError: `Cyclic dependency in move statements: The following chained move statements form a cycle, and so there is no final location to move objects to:
+  - test:1,1: module.nonexist → module.count[0]
+  - test:1,1: module.nonexist.test.single → module.count[0].test.count[0]
+
+A chain of move statements must end with an address that doesn't appear in any other statements, and which typically also refers to an object still declared in the configuration.`,
+		},
+		"fully contained nested statements": {
+			// we have to avoid a cycle because the nested moves appear in both
+			// the from and to address of the parent when only the module index
+			// is changing.
+			Statements: []MoveStatement{
+				makeTestMoveStmt(t, `count`,
+					`test.count`,
+					`test.count[0]`,
+				),
+				makeTestMoveStmt(t, ``,
+					`module.count`,
+					`module.count[0]`,
+				),
+			},
+		},
+		"double fully contained nested statements": {
+			// we have to avoid a cycle because the nested moves appear in both
+			// the from and to address of the parent when only the module index
+			// is changing.
+			Statements: []MoveStatement{
+				makeTestMoveStmt(t, `count`,
+					`module.count`,
+					`module.count[0]`,
+				),
+				makeTestMoveStmt(t, `count.count`,
+					`test.count`,
+					`test.count[0]`,
+				),
+				makeTestMoveStmt(t, ``,
+					`module.count`,
+					`module.count[0]`,
+				),
+			},
+		},
 	}
 
 	for name, test := range tests {
@@ -598,6 +694,13 @@ func makeTestMoveStmt(t *testing.T, moduleStr, fromStr, toStr string) MoveStatem
 	}
 }
 
+func makeTestImpliedMoveStmt(t *testing.T, moduleStr, fromStr, toStr string) MoveStatement {
+	t.Helper()
+	ret := makeTestMoveStmt(t, moduleStr, fromStr, toStr)
+	ret.Implied = true
+	return ret
+}
+
 var fakeExternalModuleSource = addrs.ModuleSourceRemote{
 	PackageAddr: addrs.ModulePackage("fake-external:///"),
 }

internal/refactoring/testdata/move-statement-implied/child/move-statement-implied.tf

@@ -0,0 +1,16 @@
+# This fixture is useful only in conjunction with a previous run state that
+# conforms to the statements encoded in the resource names. It's for
+# TestImpliedMoveStatements only.
+
+resource "foo" "formerly_count" {
+  # but not count anymore
+}
+
+resource "foo" "now_count" {
+  count = 1
+}
+
+moved {
+  from = foo.no_longer_present[1]
+  to   = foo.no_longer_present
+}

internal/refactoring/testdata/move-statement-implied/move-statement-implied.tf

@@ -48,3 +48,7 @@ resource "foo" "ambiguous" {
   # set it up to have both no-key and zero-key instances in the
   # state.
 }
+
+module "child" {
+  source = "./child"
+}

internal/states/instance_object.go

@@ -1,6 +1,8 @@
 package states
 
 import (
+	"sort"
+
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 
@@ -108,6 +110,13 @@ func (o *ResourceInstanceObject) Encode(ty cty.Type, schemaVersion uint64) (*Res
 		return nil, err
 	}
 
+	// Dependencies are collected and merged in an unordered format (using map
+	// keys as a set), then later changed to a slice (in random ordering) to be
+	// stored in state as an array. To avoid pointless thrashing of state in
+	// refresh-only runs, we can either override comparison of dependency lists
+	// (more desirable, but tricky for Reasons) or just sort when encoding.
+	sort.Slice(o.Dependencies, func(i, j int) bool { return o.Dependencies[i].String() < o.Dependencies[j].String() })
+
 	return &ResourceInstanceObjectSrc{
 		SchemaVersion:       schemaVersion,
 		AttrsJSON:           src,

internal/states/instance_object_test.go

@@ -0,0 +1,50 @@
+package states
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/terraform/internal/addrs"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestResourceInstanceObject_encode(t *testing.T) {
+	value := cty.ObjectVal(map[string]cty.Value{
+		"foo": cty.True,
+	})
+	// The in-memory order of resource dependencies is random, since they're an
+	// unordered set.
+	depsOne := []addrs.ConfigResource{
+		addrs.RootModule.Resource(addrs.ManagedResourceMode, "test", "honk"),
+		addrs.RootModule.Child("child").Resource(addrs.ManagedResourceMode, "test", "flub"),
+		addrs.RootModule.Resource(addrs.ManagedResourceMode, "test", "boop"),
+	}
+	depsTwo := []addrs.ConfigResource{
+		addrs.RootModule.Child("child").Resource(addrs.ManagedResourceMode, "test", "flub"),
+		addrs.RootModule.Resource(addrs.ManagedResourceMode, "test", "boop"),
+		addrs.RootModule.Resource(addrs.ManagedResourceMode, "test", "honk"),
+	}
+	rioOne := &ResourceInstanceObject{
+		Value:        value,
+		Status:       ObjectPlanned,
+		Dependencies: depsOne,
+	}
+	rioTwo := &ResourceInstanceObject{
+		Value:        value,
+		Status:       ObjectPlanned,
+		Dependencies: depsTwo,
+	}
+	riosOne, err := rioOne.Encode(value.Type(), 0)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	riosTwo, err := rioTwo.Encode(value.Type(), 0)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	// However, identical sets of dependencies should always be written to state
+	// in an identical order, so we don't do meaningless state updates on refresh.
+	if diff := cmp.Diff(riosOne.Dependencies, riosTwo.Dependencies); diff != "" {
+		t.Errorf("identical dependencies got encoded in different orders:\n%s", diff)
+	}
+}

internal/terraform/context_apply.go

@@ -30,30 +30,11 @@ func (c *Context) Apply(plan *plans.Plan, config *configs.Config) (*states.State
 		return nil, diags
 	}
 
-	variables := InputValues{}
-	for name, dyVal := range plan.VariableValues {
-		val, err := dyVal.Decode(cty.DynamicPseudoType)
-		if err != nil {
-			diags = diags.Append(tfdiags.Sourceless(
-				tfdiags.Error,
-				"Invalid variable value in plan",
-				fmt.Sprintf("Invalid value for variable %q recorded in plan file: %s.", name, err),
-			))
-			continue
-		}
-
-		variables[name] = &InputValue{
-			Value:      val,
-			SourceType: ValueFromPlan,
-		}
-	}
-
 	workingState := plan.PriorState.DeepCopy()
 	walker, walkDiags := c.walk(graph, operation, &graphWalkOpts{
 		Config:     config,
 		InputState: workingState,
 		Changes:    plan.Changes,
-		RootVariableValues: variables,
 	})
 	diags = diags.Append(walker.NonFatalDiagnostics)
 	diags = diags.Append(walkDiags)
@@ -83,15 +64,58 @@ Note that the -target option is not suitable for routine use, and is provided on
 }
 
 func (c *Context) applyGraph(plan *plans.Plan, config *configs.Config, validate bool) (*Graph, walkOperation, tfdiags.Diagnostics) {
-	graph, diags := (&ApplyGraphBuilder{
+	var diags tfdiags.Diagnostics
+
+	variables := InputValues{}
+	for name, dyVal := range plan.VariableValues {
+		val, err := dyVal.Decode(cty.DynamicPseudoType)
+		if err != nil {
+			diags = diags.Append(tfdiags.Sourceless(
+				tfdiags.Error,
+				"Invalid variable value in plan",
+				fmt.Sprintf("Invalid value for variable %q recorded in plan file: %s.", name, err),
+			))
+			continue
+		}
+
+		variables[name] = &InputValue{
+			Value:      val,
+			SourceType: ValueFromPlan,
+		}
+	}
+	if diags.HasErrors() {
+		return nil, walkApply, diags
+	}
+
+	// The plan.VariableValues field only records variables that were actually
+	// set by the caller in the PlanOpts, so we may need to provide
+	// placeholders for any other variables that the user didn't set, in
+	// which case Terraform will once again use the default value from the
+	// configuration when we visit these variables during the graph walk.
+	for name := range config.Module.Variables {
+		if _, ok := variables[name]; ok {
+			continue
+		}
+		variables[name] = &InputValue{
+			Value:      cty.NilVal,
+			SourceType: ValueFromPlan,
+		}
+	}
+
+	graph, moreDiags := (&ApplyGraphBuilder{
 		Config:             config,
 		Changes:            plan.Changes,
 		State:              plan.PriorState,
+		RootVariableValues: variables,
 		Plugins:            c.plugins,
 		Targets:            plan.TargetAddrs,
 		ForceReplace:       plan.ForceReplaceAddrs,
 		Validate:           validate,
 	}).Build(addrs.RootModuleInstance)
+	diags = diags.Append(moreDiags)
+	if moreDiags.HasErrors() {
+		return nil, walkApply, diags
+	}
 
 	operation := walkApply
 	if plan.UIMode == plans.DestroyMode {

internal/terraform/context_apply2_test.go

@@ -426,7 +426,7 @@ resource "test_resource" "b" {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	_, diags = ctx.Apply(plan, m)
@@ -530,14 +530,14 @@ resource "test_object" "y" {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
 	assertNoErrors(t, diags)
 
 	// FINAL PLAN:
-	plan, diags = ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags = ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	// make sure the same marks are compared in the next plan as well

internal/terraform/context_apply_test.go

@@ -517,7 +517,7 @@ func TestContext2Apply_mapVarBetweenModules(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -2262,7 +2262,7 @@ func TestContext2Apply_countVariable(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -2288,7 +2288,7 @@ func TestContext2Apply_countVariableRef(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -2327,7 +2327,7 @@ func TestContext2Apply_provisionerInterpCount(t *testing.T) {
 		Provisioners: provisioners,
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	// We'll marshal and unmarshal the plan here, to ensure that we have
@@ -3682,7 +3682,7 @@ func TestContext2Apply_multiVarOrder(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -3713,7 +3713,7 @@ func TestContext2Apply_multiVarOrderInterp(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -4704,9 +4704,7 @@ func TestContext2Apply_provisionerDestroy(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, state, &PlanOpts{
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.DestroyMode, testInputValuesUnset(m.Module.Variables)))
-		Mode: plans.DestroyMode,
-	})
 	assertNoErrors(t, diags)
 
 	state, diags = ctx.Apply(plan, m)
@@ -4753,9 +4751,7 @@ func TestContext2Apply_provisionerDestroyFail(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, state, &PlanOpts{
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.DestroyMode, testInputValuesUnset(m.Module.Variables)))
-		Mode: plans.DestroyMode,
-	})
 	assertNoErrors(t, diags)
 
 	state, diags = ctx.Apply(plan, m)
@@ -5908,7 +5904,7 @@ func TestContext2Apply_destroyWithModuleVariableAndCountNested(t *testing.T) {
 		})
 
 		// First plan and apply a create operation
-		plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+		plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 		assertNoErrors(t, diags)
 
 		state, diags = ctx.Apply(plan, m)
@@ -5929,9 +5925,7 @@ func TestContext2Apply_destroyWithModuleVariableAndCountNested(t *testing.T) {
 		})
 
 		// First plan and apply a create operation
-		plan, diags := ctx.Plan(m, state, &PlanOpts{
+		plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.DestroyMode, testInputValuesUnset(m.Module.Variables)))
-			Mode: plans.DestroyMode,
-		})
 		if diags.HasErrors() {
 			t.Fatalf("destroy plan err: %s", diags.Err())
 		}
@@ -7561,6 +7555,12 @@ func TestContext2Apply_vars(t *testing.T) {
 			Value:      cty.StringVal("us-east-1"),
 			SourceType: ValueFromCaller,
 		},
+		"bar": &InputValue{
+			// This one is not explicitly set but that's okay because it
+			// has a declared default, which Terraform Core will use instead.
+			Value:      cty.NilVal,
+			SourceType: ValueFromCaller,
+		},
 		"test_list": &InputValue{
 			Value: cty.ListVal([]cty.Value{
 				cty.StringVal("Hello"),
@@ -7876,7 +7876,7 @@ func TestContext2Apply_issue7824(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("err: %s", diags.Err())
 	}
@@ -7932,7 +7932,7 @@ func TestContext2Apply_issue5254(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("err: %s", diags.Err())
 	}
@@ -7951,7 +7951,7 @@ func TestContext2Apply_issue5254(t *testing.T) {
 		},
 	})
 
-	plan, diags = ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags = ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("err: %s", diags.Err())
 	}
@@ -8845,7 +8845,7 @@ func TestContext2Apply_plannedInterpolatedCount(t *testing.T) {
 		Providers: Providers,
 	})
 
-	plan, diags := ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("plan failed: %s", diags.Err())
 	}
@@ -8904,9 +8904,7 @@ func TestContext2Apply_plannedDestroyInterpolatedCount(t *testing.T) {
 		Providers: providers,
 	})
 
-	plan, diags := ctx.Plan(m, state, &PlanOpts{
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.DestroyMode, testInputValuesUnset(m.Module.Variables)))
-		Mode: plans.DestroyMode,
-	})
 	if diags.HasErrors() {
 		t.Fatalf("plan failed: %s", diags.Err())
 	}
@@ -9674,7 +9672,7 @@ func TestContext2Apply_plannedConnectionRefs(t *testing.T) {
 		Hooks:        []Hook{hook},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	diags.HasErrors()
 	if diags.HasErrors() {
 		t.Fatalf("diags: %s", diags.Err())
@@ -11687,7 +11685,7 @@ resource "test_resource" "foo" {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags := ctx.Apply(plan, m)
@@ -11702,7 +11700,7 @@ resource "test_resource" "foo" {
 		},
 	})
 
-	plan, diags = ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags = ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	state, diags = ctx.Apply(plan, m)
@@ -11720,6 +11718,7 @@ resource "test_resource" "foo" {
 	plan, diags = ctx.Plan(m, state, &PlanOpts{
 		Mode: plans.NormalMode,
 		SetVariables: InputValues{
+			"sensitive_id": &InputValue{Value: cty.NilVal},
 			"sensitive_var": &InputValue{
 				Value: cty.StringVal("bar"),
 			},
@@ -11759,7 +11758,7 @@ resource "test_resource" "foo" {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("plan errors: %s", diags.Err())
 	}
@@ -11904,7 +11903,7 @@ resource "test_resource" "foo" {
 		)
 	})
 
-	plan, diags := ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	addr := mustResourceInstanceAddr("test_resource.foo")
@@ -11954,7 +11953,7 @@ resource "test_resource" "foo" {
 	// but this seems rather suspicious and we should ideally figure out what
 	// this test was originally intending to do and make it do that.
 	oldPlan := plan
-	_, diags = ctx2.Plan(m2, state, DefaultPlanOpts)
+	_, diags = ctx2.Plan(m2, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 	stateWithoutSensitive, diags := ctx.Apply(oldPlan, m)
 	assertNoErrors(t, diags)
@@ -12206,7 +12205,7 @@ func TestContext2Apply_dataSensitive(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("diags: %s", diags.Err())
 	} else {

internal/terraform/context_eval.go

@@ -45,7 +45,7 @@ func (c *Context) Eval(config *configs.Config, state *states.State, moduleAddr a
 	state = state.DeepCopy()
 	var walker *ContextGraphWalker
 
-	variables := mergeDefaultInputVariableValues(opts.SetVariables, config.Module.Variables)
+	variables := opts.SetVariables
 
 	// By the time we get here, we should have values defined for all of
 	// the root module variables, even if some of them are "unknown". It's the
@@ -62,6 +62,7 @@ func (c *Context) Eval(config *configs.Config, state *states.State, moduleAddr a
 	graph, moreDiags := (&EvalGraphBuilder{
 		Config:             config,
 		State:              state,
+		RootVariableValues: variables,
 		Plugins:            c.plugins,
 	}).Build(addrs.RootModuleInstance)
 	diags = diags.Append(moreDiags)
@@ -72,7 +73,6 @@ func (c *Context) Eval(config *configs.Config, state *states.State, moduleAddr a
 	walkOpts := &graphWalkOpts{
 		InputState: state,
 		Config:     config,
-		RootVariableValues: variables,
 	}
 
 	walker, moreDiags = c.walk(graph, walkEval, walkOpts)

internal/terraform/context_eval_test.go

@@ -54,7 +54,9 @@ func TestContextEval(t *testing.T) {
 		},
 	})
 
-	scope, diags := ctx.Eval(m, states.NewState(), addrs.RootModuleInstance, &EvalOpts{})
+	scope, diags := ctx.Eval(m, states.NewState(), addrs.RootModuleInstance, &EvalOpts{
+		SetVariables: testInputValuesUnset(m.Module.Variables),
+	})
 	if diags.HasErrors() {
 		t.Fatalf("Eval errors: %s", diags.Err())
 	}

internal/terraform/context_import.go

@@ -53,10 +53,13 @@ func (c *Context) Import(config *configs.Config, prevRunState *states.State, opt
 
 	log.Printf("[DEBUG] Building and walking import graph")
 
+	variables := opts.SetVariables
+
 	// Initialize our graph builder
 	builder := &ImportGraphBuilder{
 		ImportTargets:      opts.Targets,
 		Config:             config,
+		RootVariableValues: variables,
 		Plugins:            c.plugins,
 	}
 
@@ -67,13 +70,10 @@ func (c *Context) Import(config *configs.Config, prevRunState *states.State, opt
 		return state, diags
 	}
 
-	variables := mergeDefaultInputVariableValues(opts.SetVariables, config.Module.Variables)
-
 	// Walk it
 	walker, walkDiags := c.walk(graph, walkImport, &graphWalkOpts{
 		Config:     config,
 		InputState: state,
-		RootVariableValues: variables,
 	})
 	diags = diags.Append(walkDiags)
 	if walkDiags.HasErrors() {

internal/terraform/context_plan.go

@@ -21,10 +21,42 @@ import (
 // PlanOpts are the various options that affect the details of how Terraform
 // will build a plan.
 type PlanOpts struct {
+	// Mode defines what variety of plan the caller wishes to create.
+	// Refer to the documentation of the plans.Mode type and its values
+	// for more information.
 	Mode plans.Mode
+
+	// SkipRefresh specifies to trust that the current values for managed
+	// resource instances in the prior state are accurate and to therefore
+	// disable the usual step of fetching updated values for each resource
+	// instance using its corresponding provider.
 	SkipRefresh bool
+
+	// SetVariables are the raw values for root module variables as provided
+	// by the user who is requesting the run, prior to any normalization or
+	// substitution of defaults. See the documentation for the InputValue
+	// type for more information on how to correctly populate this.
 	SetVariables InputValues
+
+	// If Targets has a non-zero length then it activates targeted planning
+	// mode, where Terraform will take actions only for resource instances
+	// mentioned in this set and any other objects those resource instances
+	// depend on.
+	//
+	// Targeted planning mode is intended for exceptional use only,
+	// and so populating this field will cause Terraform to generate extra
+	// warnings as part of the planning result.
 	Targets []addrs.Targetable
+
+	// ForceReplace is a set of resource instance addresses whose corresponding
+	// objects should be forced planned for replacement if the provider's
+	// plan would otherwise have been to either update the object in-place or
+	// to take no action on it at all.
+	//
+	// A typical use of this argument is to ask Terraform to replace an object
+	// which the user has determined is somehow degraded (via information from
+	// outside of Terraform), thereby hopefully replacing it with a
+	// fully-functional new object.
 	ForceReplace []addrs.AbsResourceInstance
 }
 
@@ -99,8 +131,6 @@ func (c *Context) Plan(config *configs.Config, prevRunState *states.State, opts
 		return nil, diags
 	}
 
-	variables := mergeDefaultInputVariableValues(opts.SetVariables, config.Module.Variables)
-
 	// By the time we get here, we should have values defined for all of
 	// the root module variables, even if some of them are "unknown". It's the
 	// caller's responsibility to have already handled the decoding of these
@@ -108,7 +138,7 @@ func (c *Context) Plan(config *configs.Config, prevRunState *states.State, opts
 	// user-friendly error messages if they are not all present, and so
 	// the error message from checkInputVariables should never be seen and
 	// includes language asking the user to report a bug.
-	varDiags := checkInputVariables(config.Module.Variables, variables)
+	varDiags := checkInputVariables(config.Module.Variables, opts.SetVariables)
 	diags = diags.Append(varDiags)
 
 	if len(opts.Targets) > 0 {
@@ -125,11 +155,11 @@ The -target option is not for routine use, and is provided only for exceptional
 	var planDiags tfdiags.Diagnostics
 	switch opts.Mode {
 	case plans.NormalMode:
-		plan, planDiags = c.plan(config, prevRunState, variables, opts)
+		plan, planDiags = c.plan(config, prevRunState, opts)
 	case plans.DestroyMode:
-		plan, planDiags = c.destroyPlan(config, prevRunState, variables, opts)
+		plan, planDiags = c.destroyPlan(config, prevRunState, opts)
 	case plans.RefreshOnlyMode:
-		plan, planDiags = c.refreshOnlyPlan(config, prevRunState, variables, opts)
+		plan, planDiags = c.refreshOnlyPlan(config, prevRunState, opts)
 	default:
 		panic(fmt.Sprintf("unsupported plan mode %s", opts.Mode))
 	}
@@ -139,8 +169,12 @@ The -target option is not for routine use, and is provided only for exceptional
 	}
 
 	// convert the variables into the format expected for the plan
-	varVals := make(map[string]plans.DynamicValue, len(variables))
+	varVals := make(map[string]plans.DynamicValue, len(opts.SetVariables))
-	for k, iv := range variables {
+	for k, iv := range opts.SetVariables {
+		if iv.Value == cty.NilVal {
+			continue // We only record values that the caller actually set
+		}
+
 		// We use cty.DynamicPseudoType here so that we'll save both the
 		// value _and_ its dynamic type in the plan, so we can recover
 		// exactly the same value later.
@@ -172,14 +206,33 @@ var DefaultPlanOpts = &PlanOpts{
 	Mode: plans.NormalMode,
 }
 
-func (c *Context) plan(config *configs.Config, prevRunState *states.State, rootVariables InputValues, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
+// SimplePlanOpts is a constructor to help with creating "simple" values of
+// PlanOpts which only specify a mode and input variables.
+//
+// This helper function is primarily intended for use in straightforward
+// tests that don't need any of the more "esoteric" planning options. For
+// handling real user requests to run Terraform, it'd probably be better
+// to construct a *PlanOpts value directly and provide a way for the user
+// to set values for all of its fields.
+//
+// The "mode" and "setVariables" arguments become the values of the "Mode"
+// and "SetVariables" fields in the result. Refer to the PlanOpts type
+// documentation to learn about the meanings of those fields.
+func SimplePlanOpts(mode plans.Mode, setVariables InputValues) *PlanOpts {
+	return &PlanOpts{
+		Mode:         mode,
+		SetVariables: setVariables,
+	}
+}
+
+func (c *Context) plan(config *configs.Config, prevRunState *states.State, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	if opts.Mode != plans.NormalMode {
 		panic(fmt.Sprintf("called Context.plan with %s", opts.Mode))
 	}
 
-	plan, walkDiags := c.planWalk(config, prevRunState, rootVariables, opts)
+	plan, walkDiags := c.planWalk(config, prevRunState, opts)
 	diags = diags.Append(walkDiags)
 	if diags.HasErrors() {
 		return nil, diags
@@ -194,14 +247,14 @@ func (c *Context) plan(config *configs.Config, prevRunState *states.State, rootV
 	return plan, diags
 }
 
-func (c *Context) refreshOnlyPlan(config *configs.Config, prevRunState *states.State, rootVariables InputValues, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
+func (c *Context) refreshOnlyPlan(config *configs.Config, prevRunState *states.State, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	if opts.Mode != plans.RefreshOnlyMode {
 		panic(fmt.Sprintf("called Context.refreshOnlyPlan with %s", opts.Mode))
 	}
 
-	plan, walkDiags := c.planWalk(config, prevRunState, rootVariables, opts)
+	plan, walkDiags := c.planWalk(config, prevRunState, opts)
 	diags = diags.Append(walkDiags)
 	if diags.HasErrors() {
 		return nil, diags
@@ -235,7 +288,7 @@ func (c *Context) refreshOnlyPlan(config *configs.Config, prevRunState *states.S
 	return plan, diags
 }
 
-func (c *Context) destroyPlan(config *configs.Config, prevRunState *states.State, rootVariables InputValues, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
+func (c *Context) destroyPlan(config *configs.Config, prevRunState *states.State, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	pendingPlan := &plans.Plan{}
 
@@ -260,7 +313,7 @@ func (c *Context) destroyPlan(config *configs.Config, prevRunState *states.State
 		log.Printf("[TRACE] Context.destroyPlan: calling Context.plan to get the effect of refreshing the prior state")
 		normalOpts := *opts
 		normalOpts.Mode = plans.NormalMode
-		refreshPlan, refreshDiags := c.plan(config, prevRunState, rootVariables, &normalOpts)
+		refreshPlan, refreshDiags := c.plan(config, prevRunState, &normalOpts)
 		if refreshDiags.HasErrors() {
 			// NOTE: Normally we'd append diagnostics regardless of whether
 			// there are errors, just in case there are warnings we'd want to
@@ -291,7 +344,7 @@ func (c *Context) destroyPlan(config *configs.Config, prevRunState *states.State
 		priorState = pendingPlan.PriorState
 	}
 
-	destroyPlan, walkDiags := c.planWalk(config, priorState, rootVariables, opts)
+	destroyPlan, walkDiags := c.planWalk(config, priorState, opts)
 	diags = diags.Append(walkDiags)
 	if walkDiags.HasErrors() {
 		return nil, diags
@@ -392,7 +445,7 @@ func (c *Context) postPlanValidateMoves(config *configs.Config, stmts []refactor
 	return refactoring.ValidateMoves(stmts, config, allInsts)
 }
 
-func (c *Context) planWalk(config *configs.Config, prevRunState *states.State, rootVariables InputValues, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
+func (c *Context) planWalk(config *configs.Config, prevRunState *states.State, opts *PlanOpts) (*plans.Plan, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	log.Printf("[DEBUG] Building and walking plan graph for %s", opts.Mode)
 
@@ -423,7 +476,6 @@ func (c *Context) planWalk(config *configs.Config, prevRunState *states.State, r
 		InputState:  prevRunState,
 		Changes:     changes,
 		MoveResults: moveResults,
-		RootVariableValues: rootVariables,
 	})
 	diags = diags.Append(walker.NonFatalDiagnostics)
 	diags = diags.Append(walkDiags)
@@ -471,6 +523,7 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
 		graph, diags := (&PlanGraphBuilder{
 			Config:             config,
 			State:              prevRunState,
+			RootVariableValues: opts.SetVariables,
 			Plugins:            c.plugins,
 			Targets:            opts.Targets,
 			ForceReplace:       opts.ForceReplace,
@@ -482,6 +535,7 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
 		graph, diags := (&PlanGraphBuilder{
 			Config:             config,
 			State:              prevRunState,
+			RootVariableValues: opts.SetVariables,
 			Plugins:            c.plugins,
 			Targets:            opts.Targets,
 			Validate:           validate,
@@ -493,6 +547,7 @@ func (c *Context) planGraph(config *configs.Config, prevRunState *states.State,
 		graph, diags := (&DestroyPlanGraphBuilder{
 			Config:             config,
 			State:              prevRunState,
+			RootVariableValues: opts.SetVariables,
 			Plugins:            c.plugins,
 			Targets:            opts.Targets,
 			Validate:           validate,

internal/terraform/context_plan2_test.go

@@ -205,7 +205,7 @@ data "test_data_source" "foo" {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	assertNoErrors(t, diags)
 
 	for _, res := range plan.Changes.Resources {

internal/terraform/context_plan_test.go

@@ -405,7 +405,7 @@ func TestContext2Plan_moduleExpand(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("unexpected errors: %s", diags.Err())
 	}
@@ -1175,7 +1175,7 @@ func TestContext2Plan_moduleProviderVar(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("unexpected errors: %s", diags.Err())
 	}
@@ -2242,7 +2242,7 @@ func TestContext2Plan_countModuleStatic(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("unexpected errors: %s", diags.Err())
 	}
@@ -2295,7 +2295,7 @@ func TestContext2Plan_countModuleStaticGrandchild(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("unexpected errors: %s", diags.Err())
 	}
@@ -3938,7 +3938,7 @@ func TestContext2Plan_taintDestroyInterpolatedCountRace(t *testing.T) {
 			},
 		})
 
-		plan, diags := ctx.Plan(m, state.DeepCopy(), DefaultPlanOpts)
+		plan, diags := ctx.Plan(m, state.DeepCopy(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 		if diags.HasErrors() {
 			t.Fatalf("unexpected errors: %s", diags.Err())
 		}
@@ -5481,7 +5481,7 @@ func TestContext2Plan_variableSensitivity(t *testing.T) {
 		},
 	})
 
-	plan, diags := ctx.Plan(m, states.NewState(), DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, states.NewState(), SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatalf("unexpected errors: %s", diags.Err())
 	}
@@ -5544,6 +5544,7 @@ func TestContext2Plan_variableSensitivityModule(t *testing.T) {
 	plan, diags := ctx.Plan(m, states.NewState(), &PlanOpts{
 		Mode: plans.NormalMode,
 		SetVariables: InputValues{
+			"sensitive_var": {Value: cty.NilVal},
 			"another_var": &InputValue{
 				Value:      cty.StringVal("boop"),
 				SourceType: ValueFromCaller,
@@ -6657,7 +6658,7 @@ resource "test_resource" "foo" {
 			},
 		)
 	})
-	plan, diags := ctx.Plan(m, state, DefaultPlanOpts)
+	plan, diags := ctx.Plan(m, state, SimplePlanOpts(plans.NormalMode, testInputValuesUnset(m.Module.Variables)))
 	if diags.HasErrors() {
 		t.Fatal(diags.Err())
 	}

internal/terraform/context_validate.go

@@ -37,17 +37,6 @@ func (c *Context) Validate(config *configs.Config) tfdiags.Diagnostics {
 
 	log.Printf("[DEBUG] Building and walking validate graph")
 
-	graph, moreDiags := ValidateGraphBuilder(&PlanGraphBuilder{
-		Config:   config,
-		Plugins:  c.plugins,
-		Validate: true,
-		State:    states.NewState(),
-	}).Build(addrs.RootModuleInstance)
-	diags = diags.Append(moreDiags)
-	if moreDiags.HasErrors() {
-		return diags
-	}
-
 	// Validate is to check if the given module is valid regardless of
 	// input values, current state, etc. Therefore we populate all of the
 	// input values with unknown values of the expected type, allowing us
@@ -66,9 +55,20 @@ func (c *Context) Validate(config *configs.Config) tfdiags.Diagnostics {
 		}
 	}
 
+	graph, moreDiags := ValidateGraphBuilder(&PlanGraphBuilder{
+		Config:             config,
+		Plugins:            c.plugins,
+		Validate:           true,
+		State:              states.NewState(),
+		RootVariableValues: varValues,
+	}).Build(addrs.RootModuleInstance)
+	diags = diags.Append(moreDiags)
+	if moreDiags.HasErrors() {
+		return diags
+	}
+
 	walker, walkDiags := c.walk(graph, walkValidate, &graphWalkOpts{
 		Config: config,
-		RootVariableValues: varValues,
 	})
 	diags = diags.Append(walker.NonFatalDiagnostics)
 	diags = diags.Append(walkDiags)

internal/terraform/context_validate_test.go

@@ -1187,32 +1187,6 @@ resource "aws_instance" "foo" {
 	}
 }
 
-// Manually validate using the new PlanGraphBuilder
-func TestContext2Validate_PlanGraphBuilder(t *testing.T) {
-	fixture := contextFixtureApplyVars(t)
-	opts := fixture.ContextOpts()
-	c := testContext2(t, opts)
-
-	graph, diags := ValidateGraphBuilder(&PlanGraphBuilder{
-		Config:  fixture.Config,
-		State:   states.NewState(),
-		Plugins: c.plugins,
-	}).Build(addrs.RootModuleInstance)
-	if diags.HasErrors() {
-		t.Fatalf("errors from PlanGraphBuilder: %s", diags.Err())
-	}
-	defer c.acquireRun("validate-test")()
-	walker, diags := c.walk(graph, walkValidate, &graphWalkOpts{
-		Config: fixture.Config,
-	})
-	if diags.HasErrors() {
-		t.Fatal(diags.Err())
-	}
-	if len(walker.NonFatalDiagnostics) > 0 {
-		t.Fatal(walker.NonFatalDiagnostics.Err())
-	}
-}
-
 func TestContext2Validate_invalidOutput(t *testing.T) {
 	m := testModuleInline(t, map[string]string{
 		"main.tf": `
@@ -2088,3 +2062,36 @@ output "out" {
 		t.Fatal(diags.ErrWithWarnings())
 	}
 }
+
+func TestContext2Validate_nonNullableVariableDefaultValidation(t *testing.T) {
+	m := testModuleInline(t, map[string]string{
+		"main.tf": `
+ module "first" {
+   source = "./mod"
+   input = null
+ }
+ `,
+
+		"mod/main.tf": `
+ variable "input" {
+   type        = string
+   default     = "default"
+   nullable    = false
+
+   // Validation expressions should receive the default with nullable=false and
+   // a null input.
+   validation {
+     condition     = var.input != null
+     error_message = "Input cannot be null!"
+   }
+ }
+ `,
+	})
+
+	ctx := testContext2(t, &ContextOpts{})
+
+	diags := ctx.Validate(m)
+	if diags.HasErrors() {
+		t.Fatal(diags.ErrWithWarnings())
+	}
+}

internal/terraform/context_walk.go

@@ -23,7 +23,6 @@ type graphWalkOpts struct {
 	Changes    *plans.Changes
 	Config     *configs.Config
 
-	RootVariableValues InputValues
 	MoveResults refactoring.MoveResults
 }
 
@@ -108,6 +107,5 @@ func (c *Context) graphWalker(operation walkOperation, opts *graphWalkOpts) *Con
 		MoveResults:      opts.MoveResults,
 		Operation:        operation,
 		StopContext:      c.runContext,
-		RootVariableValues: opts.RootVariableValues,
 	}
 }

internal/terraform/eval_context.go

@@ -121,12 +121,24 @@ type EvalContext interface {
 	// addresses in this context.
 	EvaluationScope(self addrs.Referenceable, keyData InstanceKeyEvalData) *lang.Scope
 
-	// SetModuleCallArguments defines values for the variables of a particular
+	// SetRootModuleArgument defines the value for one variable of the root
-	// child module call.
+	// module. The caller must ensure that given value is a suitable
+	// "final value" for the variable, which means that it's already converted
+	// and validated to match any configured constraints and validation rules.
 	//
-	// Calling this function multiple times has merging behavior, keeping any
+	// Calling this function multiple times with the same variable address
-	// previously-set keys that are not present in the new map.
+	// will silently overwrite the value provided by a previous call.
-	SetModuleCallArguments(addrs.ModuleCallInstance, map[string]cty.Value)
+	SetRootModuleArgument(addrs.InputVariable, cty.Value)
+
+	// SetModuleCallArgument defines the value for one input variable of a
+	// particular child module call. The caller must ensure that the given
+	// value is a suitable "final value" for the variable, which means that
+	// it's already converted and validated to match any configured
+	// constraints and validation rules.
+	//
+	// Calling this function multiple times with the same variable address
+	// will silently overwrite the value provided by a previous call.
+	SetModuleCallArgument(addrs.ModuleCallInstance, addrs.InputVariable, cty.Value)
 
 	// GetVariableValue returns the value provided for the input variable with
 	// the given address, or cty.DynamicVal if the variable hasn't been assigned

internal/terraform/eval_context_builtin.go

@@ -313,7 +313,21 @@ func (ctx *BuiltinEvalContext) Path() addrs.ModuleInstance {
 	return ctx.PathValue
 }
 
-func (ctx *BuiltinEvalContext) SetModuleCallArguments(n addrs.ModuleCallInstance, vals map[string]cty.Value) {
+func (ctx *BuiltinEvalContext) SetRootModuleArgument(addr addrs.InputVariable, v cty.Value) {
+	ctx.VariableValuesLock.Lock()
+	defer ctx.VariableValuesLock.Unlock()
+
+	log.Printf("[TRACE] BuiltinEvalContext: Storing final value for variable %s", addr.Absolute(addrs.RootModuleInstance))
+	key := addrs.RootModuleInstance.String()
+	args := ctx.VariableValues[key]
+	if args == nil {
+		args = make(map[string]cty.Value)
+		ctx.VariableValues[key] = args
+	}
+	args[addr.Name] = v
+}
+
+func (ctx *BuiltinEvalContext) SetModuleCallArgument(callAddr addrs.ModuleCallInstance, varAddr addrs.InputVariable, v cty.Value) {
 	ctx.VariableValuesLock.Lock()
 	defer ctx.VariableValuesLock.Unlock()
 
@@ -321,18 +335,15 @@ func (ctx *BuiltinEvalContext) SetModuleCallArguments(n addrs.ModuleCallInstance
 		panic("context path not set")
 	}
 
-	childPath := n.ModuleInstance(ctx.PathValue)
+	childPath := callAddr.ModuleInstance(ctx.PathValue)
+	log.Printf("[TRACE] BuiltinEvalContext: Storing final value for variable %s", varAddr.Absolute(childPath))
 	key := childPath.String()
-
 	args := ctx.VariableValues[key]
 	if args == nil {
-		ctx.VariableValues[key] = vals
+		args = make(map[string]cty.Value)
-		return
+		ctx.VariableValues[key] = args
-	}
-
-	for k, v := range vals {
-		args[k] = v
 	}
+	args[varAddr.Name] = v
 }
 
 func (ctx *BuiltinEvalContext) GetVariableValue(addr addrs.AbsInputVariableInstance) cty.Value {

internal/terraform/eval_context_mock.go

@@ -111,13 +111,21 @@ type MockEvalContext struct {
 	PathCalled bool
 	PathPath   addrs.ModuleInstance
 
-	SetModuleCallArgumentsCalled bool
+	SetRootModuleArgumentCalled bool
-	SetModuleCallArgumentsModule addrs.ModuleCallInstance
+	SetRootModuleArgumentAddr   addrs.InputVariable
-	SetModuleCallArgumentsValues map[string]cty.Value
+	SetRootModuleArgumentValue  cty.Value
+	SetRootModuleArgumentFunc   func(addr addrs.InputVariable, v cty.Value)
+
+	SetModuleCallArgumentCalled     bool
+	SetModuleCallArgumentModuleCall addrs.ModuleCallInstance
+	SetModuleCallArgumentVariable   addrs.InputVariable
+	SetModuleCallArgumentValue      cty.Value
+	SetModuleCallArgumentFunc       func(callAddr addrs.ModuleCallInstance, varAddr addrs.InputVariable, v cty.Value)
 
 	GetVariableValueCalled bool
 	GetVariableValueAddr   addrs.AbsInputVariableInstance
 	GetVariableValueValue  cty.Value
+	GetVariableValueFunc   func(addr addrs.AbsInputVariableInstance) cty.Value // supersedes GetVariableValueValue
 
 	ChangesCalled  bool
 	ChangesChanges *plans.ChangesSync
@@ -321,15 +329,31 @@ func (c *MockEvalContext) Path() addrs.ModuleInstance {
 	return c.PathPath
 }
 
-func (c *MockEvalContext) SetModuleCallArguments(n addrs.ModuleCallInstance, values map[string]cty.Value) {
+func (c *MockEvalContext) SetRootModuleArgument(addr addrs.InputVariable, v cty.Value) {
-	c.SetModuleCallArgumentsCalled = true
+	c.SetRootModuleArgumentCalled = true
-	c.SetModuleCallArgumentsModule = n
+	c.SetRootModuleArgumentAddr = addr
-	c.SetModuleCallArgumentsValues = values
+	c.SetRootModuleArgumentValue = v
+	if c.SetRootModuleArgumentFunc != nil {
+		c.SetRootModuleArgumentFunc(addr, v)
+	}
+}
+
+func (c *MockEvalContext) SetModuleCallArgument(callAddr addrs.ModuleCallInstance, varAddr addrs.InputVariable, v cty.Value) {
+	c.SetModuleCallArgumentCalled = true
+	c.SetModuleCallArgumentModuleCall = callAddr
+	c.SetModuleCallArgumentVariable = varAddr
+	c.SetModuleCallArgumentValue = v
+	if c.SetModuleCallArgumentFunc != nil {
+		c.SetModuleCallArgumentFunc(callAddr, varAddr, v)
+	}
 }
 
 func (c *MockEvalContext) GetVariableValue(addr addrs.AbsInputVariableInstance) cty.Value {
 	c.GetVariableValueCalled = true
 	c.GetVariableValueAddr = addr
+	if c.GetVariableValueFunc != nil {
+		return c.GetVariableValueFunc(addr)
+	}
 	return c.GetVariableValueValue
 }
 

internal/terraform/eval_for_each.go

@@ -78,6 +78,9 @@ func evaluateForEachExpressionValue(expr hcl.Expression, ctx EvalContext, allowU
 	}
 	ty := forEachVal.Type()
 
+	const errInvalidUnknownDetailMap = "The \"for_each\" map includes keys derived from resource attributes that cannot be determined until apply, and so Terraform cannot determine the full set of keys that will identify the instances of this resource.\n\nWhen working with unknown values in for_each, it's better to define the map keys statically in your configuration and place apply-time results only in the map values.\n\nAlternatively, you could use the -target planning option to first apply only the resources that the for_each value depends on, and then apply a second time to fully converge."
+	const errInvalidUnknownDetailSet = "The \"for_each\" set includes values derived from resource attributes that cannot be determined until apply, and so Terraform cannot determine the full set of keys that will identify the instances of this resource.\n\nWhen working with unknown values in for_each, it's better to use a map value where the keys are defined statically in your configuration and where only the values contain apply-time results.\n\nAlternatively, you could use the -target planning option to first apply only the resources that the for_each value depends on, and then apply a second time to fully converge."
+
 	switch {
 	case forEachVal.IsNull():
 		diags = diags.Append(&hcl.Diagnostic{
@@ -91,10 +94,18 @@ func evaluateForEachExpressionValue(expr hcl.Expression, ctx EvalContext, allowU
 		return nullMap, diags
 	case !forEachVal.IsKnown():
 		if !allowUnknown {
+			var detailMsg string
+			switch {
+			case ty.IsSetType():
+				detailMsg = errInvalidUnknownDetailSet
+			default:
+				detailMsg = errInvalidUnknownDetailMap
+			}
+
 			diags = diags.Append(&hcl.Diagnostic{
 				Severity:    hcl.DiagError,
 				Summary:     "Invalid for_each argument",
-				Detail:      errInvalidForEachUnknownDetail,
+				Detail:      detailMsg,
 				Subject:     expr.Range().Ptr(),
 				Expression:  expr,
 				EvalContext: hclCtx,
@@ -129,7 +140,7 @@ func evaluateForEachExpressionValue(expr hcl.Expression, ctx EvalContext, allowU
 				diags = diags.Append(&hcl.Diagnostic{
 					Severity:    hcl.DiagError,
 					Summary:     "Invalid for_each argument",
-					Detail:      errInvalidForEachUnknownDetail,
+					Detail:      errInvalidUnknownDetailSet,
 					Subject:     expr.Range().Ptr(),
 					Expression:  expr,
 					EvalContext: hclCtx,
@@ -172,8 +183,6 @@ func evaluateForEachExpressionValue(expr hcl.Expression, ctx EvalContext, allowU
 	return forEachVal, nil
 }
 
-const errInvalidForEachUnknownDetail = `The "for_each" value depends on resource attributes that cannot be determined until apply, so Terraform cannot predict how many instances will be created. To work around this, use the -target argument to first apply only the resources that the for_each depends on.`
-
 // markSafeLengthInt allows calling LengthInt on marked values safely
 func markSafeLengthInt(val cty.Value) int {
 	v, _ := val.UnmarkDeep()

internal/terraform/eval_for_each_test.go

@@ -114,12 +114,12 @@ func TestEvaluateForEachExpression_errors(t *testing.T) {
 		"unknown string set": {
 			hcltest.MockExprLiteral(cty.UnknownVal(cty.Set(cty.String))),
 			"Invalid for_each argument",
-			"depends on resource attributes that cannot be determined until apply",
+			"set includes values derived from resource attributes that cannot be determined until apply",
 		},
 		"unknown map": {
 			hcltest.MockExprLiteral(cty.UnknownVal(cty.Map(cty.Bool))),
 			"Invalid for_each argument",
-			"depends on resource attributes that cannot be determined until apply",
+			"map includes keys derived from resource attributes that cannot be determined until apply",
 		},
 		"marked map": {
 			hcltest.MockExprLiteral(cty.MapVal(map[string]cty.Value{
@@ -142,12 +142,12 @@ func TestEvaluateForEachExpression_errors(t *testing.T) {
 		"set containing unknown value": {
 			hcltest.MockExprLiteral(cty.SetVal([]cty.Value{cty.UnknownVal(cty.String)})),
 			"Invalid for_each argument",
-			"depends on resource attributes that cannot be determined until apply",
+			"set includes values derived from resource attributes that cannot be determined until apply",
 		},
 		"set containing dynamic unknown value": {
 			hcltest.MockExprLiteral(cty.SetVal([]cty.Value{cty.UnknownVal(cty.DynamicPseudoType)})),
 			"Invalid for_each argument",
-			"depends on resource attributes that cannot be determined until apply",
+			"set includes values derived from resource attributes that cannot be determined until apply",
 		},
 		"set containing marked values": {
 			hcltest.MockExprLiteral(cty.SetVal([]cty.Value{cty.StringVal("beep").Mark(marks.Sensitive), cty.StringVal("boop")})),
@@ -169,10 +169,10 @@ func TestEvaluateForEachExpression_errors(t *testing.T) {
 				t.Errorf("wrong diagnostic severity %#v; want %#v", got, want)
 			}
 			if got, want := diags[0].Description().Summary, test.Summary; got != want {
-				t.Errorf("wrong diagnostic summary %#v; want %#v", got, want)
+				t.Errorf("wrong diagnostic summary\ngot:  %s\nwant: %s", got, want)
 			}
 			if got, want := diags[0].Description().Detail, test.DetailSubstring; !strings.Contains(got, want) {
-				t.Errorf("wrong diagnostic detail %#v; want %#v", got, want)
+				t.Errorf("wrong diagnostic detail\ngot: %s\nwant substring: %s", got, want)
 			}
 			if fromExpr := diags[0].FromExpr(); fromExpr != nil {
 				if fromExpr.Expression == nil {

internal/terraform/eval_variable.go

@@ -12,6 +12,156 @@ import (
 	"github.com/zclconf/go-cty/cty/convert"
 )
 
+func prepareFinalInputVariableValue(addr addrs.AbsInputVariableInstance, raw *InputValue, cfg *configs.Variable) (cty.Value, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	convertTy := cfg.ConstraintType
+	log.Printf("[TRACE] prepareFinalInputVariableValue: preparing %s", addr)
+
+	var defaultVal cty.Value
+	if cfg.Default != cty.NilVal {
+		log.Printf("[TRACE] prepareFinalInputVariableValue: %s has a default value", addr)
+		var err error
+		defaultVal, err = convert.Convert(cfg.Default, convertTy)
+		if err != nil {
+			// Validation of the declaration should typically catch this,
+			// but we'll check it here too to be robust.
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid default value for module argument",
+				Detail: fmt.Sprintf(
+					"The default value for variable %q is incompatible with its type constraint: %s.",
+					cfg.Name, err,
+				),
+				Subject: &cfg.DeclRange,
+			})
+			// We'll return a placeholder unknown value to avoid producing
+			// redundant downstream errors.
+			return cty.UnknownVal(cfg.Type), diags
+		}
+	}
+
+	var sourceRange tfdiags.SourceRange
+	var nonFileSource string
+	if raw.HasSourceRange() {
+		sourceRange = raw.SourceRange
+	} else {
+		// If the value came from a place that isn't a file and thus doesn't
+		// have its own source range, we'll use the declaration range as
+		// our source range and generate some slightly different error
+		// messages.
+		sourceRange = tfdiags.SourceRangeFromHCL(cfg.DeclRange)
+		switch raw.SourceType {
+		case ValueFromCLIArg:
+			nonFileSource = fmt.Sprintf("set using -var=\"%s=...\"", addr.Variable.Name)
+		case ValueFromEnvVar:
+			nonFileSource = fmt.Sprintf("set using the TF_VAR_%s environment variable", addr.Variable.Name)
+		case ValueFromInput:
+			nonFileSource = "set using an interactive prompt"
+		default:
+			nonFileSource = "set from outside of the configuration"
+		}
+	}
+
+	given := raw.Value
+	if given == cty.NilVal { // The variable wasn't set at all (even to null)
+		log.Printf("[TRACE] prepareFinalInputVariableValue: %s has no defined value", addr)
+		if cfg.Required() {
+			// NOTE: The CLI layer typically checks for itself whether all of
+			// the required _root_ module variables are set, which would
+			// mask this error with a more specific one that refers to the
+			// CLI features for setting such variables. We can get here for
+			// child module variables, though.
+			log.Printf("[ERROR] prepareFinalInputVariableValue: %s is required but is not set", addr)
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  `Required variable not set`,
+				Detail:   fmt.Sprintf(`The variable %q is required, but is not set.`, addr.Variable.Name),
+				Subject:  cfg.DeclRange.Ptr(),
+			})
+			// We'll return a placeholder unknown value to avoid producing
+			// redundant downstream errors.
+			return cty.UnknownVal(cfg.Type), diags
+		}
+
+		given = defaultVal // must be set, because we checked above that the variable isn't required
+	}
+
+	val, err := convert.Convert(given, convertTy)
+	if err != nil {
+		log.Printf("[ERROR] prepareFinalInputVariableValue: %s has unsuitable type\n  got:  %s\n  want: %s", addr, given.Type(), convertTy)
+		if nonFileSource != "" {
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid value for input variable",
+				Detail: fmt.Sprintf(
+					"Unsuitable value for %s %s: %s.",
+					addr, nonFileSource, err,
+				),
+				Subject: cfg.DeclRange.Ptr(),
+			})
+		} else {
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "Invalid value for input variable",
+				Detail: fmt.Sprintf(
+					"The given value is not suitable for %s declared at %s: %s.",
+					addr, cfg.DeclRange.String(), err,
+				),
+				Subject: sourceRange.ToHCL().Ptr(),
+			})
+		}
+		// We'll return a placeholder unknown value to avoid producing
+		// redundant downstream errors.
+		return cty.UnknownVal(cfg.Type), diags
+	}
+
+	// By the time we get here, we know:
+	// - val matches the variable's type constraint
+	// - val is definitely not cty.NilVal, but might be a null value if the given was already null.
+	//
+	// That means we just need to handle the case where the value is null,
+	// which might mean we need to use the default value, or produce an error.
+	//
+	// For historical reasons we do this only for a "non-nullable" variable.
+	// Nullable variables just appear as null if they were set to null,
+	// regardless of any default value.
+	if val.IsNull() && !cfg.Nullable {
+		log.Printf("[TRACE] prepareFinalInputVariableValue: %s is defined as null", addr)
+		if defaultVal != cty.NilVal {
+			val = defaultVal
+		} else {
+			log.Printf("[ERROR] prepareFinalInputVariableValue: %s is non-nullable but set to null, and is required", addr)
+			if nonFileSource != "" {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  `Required variable not set`,
+					Detail: fmt.Sprintf(
+						"Unsuitable value for %s %s: required variable may not be set to null.",
+						addr, nonFileSource,
+					),
+					Subject: cfg.DeclRange.Ptr(),
+				})
+			} else {
+				diags = diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  `Required variable not set`,
+					Detail: fmt.Sprintf(
+						"The given value is not suitable for %s defined at %s: required variable may not be set to null.",
+						addr, cfg.DeclRange.String(),
+					),
+					Subject: sourceRange.ToHCL().Ptr(),
+				})
+			}
+			// Stub out our return value so that the semantic checker doesn't
+			// produce redundant downstream errors.
+			val = cty.UnknownVal(cfg.Type)
+		}
+	}
+
+	return val, diags
+}
+
 // evalVariableValidations ensures that all of the configured custom validations
 // for a variable are passing.
 //
@@ -20,9 +170,10 @@ import (
 // EvalModuleCallArgument for variables in descendent modules.
 func evalVariableValidations(addr addrs.AbsInputVariableInstance, config *configs.Variable, expr hcl.Expression, ctx EvalContext) (diags tfdiags.Diagnostics) {
 	if config == nil || len(config.Validations) == 0 {
-		log.Printf("[TRACE] evalVariableValidations: not active for %s, so skipping", addr)
+		log.Printf("[TRACE] evalVariableValidations: no validation rules declared for %s, so skipping", addr)
 		return nil
 	}
+	log.Printf("[TRACE] evalVariableValidations: validating %s", addr)
 
 	// Variable nodes evaluate in the parent module to where they were declared
 	// because the value expression (n.Expr, if set) comes from the calling
@@ -34,6 +185,14 @@ func evalVariableValidations(addr addrs.AbsInputVariableInstance, config *config
 	// evaluation context containing just the required value, and thus avoid
 	// the problem that ctx's evaluation functions refer to the wrong module.
 	val := ctx.GetVariableValue(addr)
+	if val == cty.NilVal {
+		diags = diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "No final value for variable",
+			Detail:   fmt.Sprintf("Terraform doesn't have a final value for %s during validation. This is a bug in Terraform; please report it!", addr),
+		})
+		return diags
+	}
 	hclCtx := &hcl.EvalContext{
 		Variables: map[string]cty.Value{
 			"var": cty.ObjectVal(map[string]cty.Value{

internal/terraform/eval_variable_test.go

@@ -0,0 +1,563 @@
+package terraform
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/hashicorp/terraform/internal/addrs"
+	"github.com/hashicorp/terraform/internal/tfdiags"
+)
+
+func TestPrepareFinalInputVariableValue(t *testing.T) {
+	// This is just a concise way to define a bunch of *configs.Variable
+	// objects to use in our tests below. We're only going to decode this
+	// config, not fully evaluate it.
+	cfgSrc := `
+		variable "nullable_required" {
+		}
+		variable "nullable_optional_default_string" {
+			default = "hello"
+		}
+		variable "nullable_optional_default_null" {
+			default = null
+		}
+		variable "constrained_string_nullable_required" {
+			type = string
+		}
+		variable "constrained_string_nullable_optional_default_string" {
+			type    = string
+			default = "hello"
+		}
+		variable "constrained_string_nullable_optional_default_bool" {
+			type    = string
+			default = true
+		}
+		variable "constrained_string_nullable_optional_default_null" {
+			type    = string
+			default = null
+		}
+		variable "required" {
+			nullable = false
+		}
+		variable "optional_default_string" {
+			nullable = false
+			default  = "hello"
+		}
+		variable "constrained_string_required" {
+			nullable = false
+			type     = string
+		}
+		variable "constrained_string_optional_default_string" {
+			nullable = false
+			type     = string
+			default  = "hello"
+		}
+		variable "constrained_string_optional_default_bool" {
+			nullable = false
+			type     = string
+			default  = true
+		}
+	`
+	cfg := testModuleInline(t, map[string]string{
+		"main.tf": cfgSrc,
+	})
+	variableConfigs := cfg.Module.Variables
+
+	// Because we loaded our pseudo-module from a temporary file, the
+	// declaration source ranges will have unpredictable filenames. We'll
+	// fix that here just to make things easier below.
+	for _, vc := range variableConfigs {
+		vc.DeclRange.Filename = "main.tf"
+	}
+
+	tests := []struct {
+		varName string
+		given   cty.Value
+		want    cty.Value
+		wantErr string
+	}{
+		// nullable_required
+		{
+			"nullable_required",
+			cty.NilVal,
+			cty.UnknownVal(cty.DynamicPseudoType),
+			`Required variable not set: The variable "nullable_required" is required, but is not set.`,
+		},
+		{
+			"nullable_required",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.DynamicPseudoType),
+			``, // "required" for a nullable variable means only that it must be set, even if it's set to null
+		},
+		{
+			"nullable_required",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"nullable_required",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// nullable_optional_default_string
+		{
+			"nullable_optional_default_string",
+			cty.NilVal,
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"nullable_optional_default_string",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.DynamicPseudoType), // nullable variables can be really set to null, masking the default
+			``,
+		},
+		{
+			"nullable_optional_default_string",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"nullable_optional_default_string",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// nullable_optional_default_null
+		{
+			"nullable_optional_default_null",
+			cty.NilVal,
+			cty.NullVal(cty.DynamicPseudoType), // the declared default value
+			``,
+		},
+		{
+			"nullable_optional_default_null",
+			cty.NullVal(cty.String),
+			cty.NullVal(cty.String), // nullable variables can be really set to null, masking the default
+			``,
+		},
+		{
+			"nullable_optional_default_null",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"nullable_optional_default_null",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_nullable_required
+		{
+			"constrained_string_nullable_required",
+			cty.NilVal,
+			cty.UnknownVal(cty.String),
+			`Required variable not set: The variable "constrained_string_nullable_required" is required, but is not set.`,
+		},
+		{
+			"constrained_string_nullable_required",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.String), // the null value still gets converted to match the type constraint
+			``,                      // "required" for a nullable variable means only that it must be set, even if it's set to null
+		},
+		{
+			"constrained_string_nullable_required",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_nullable_required",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_nullable_optional_default_string
+		{
+			"constrained_string_nullable_optional_default_string",
+			cty.NilVal,
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_string",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.String), // nullable variables can be really set to null, masking the default
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_string",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_string",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_nullable_optional_default_bool
+		{
+			"constrained_string_nullable_optional_default_bool",
+			cty.NilVal,
+			cty.StringVal("true"), // the declared default value, automatically converted to match type constraint
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_bool",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.String), // nullable variables can be really set to null, masking the default
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_bool",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_bool",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_nullable_optional_default_null
+		{
+			"constrained_string_nullable_optional_default_null",
+			cty.NilVal,
+			cty.NullVal(cty.String),
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_null",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.NullVal(cty.String),
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_null",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_nullable_optional_default_null",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// required
+		{
+			"required",
+			cty.NilVal,
+			cty.UnknownVal(cty.DynamicPseudoType),
+			`Required variable not set: The variable "required" is required, but is not set.`,
+		},
+		{
+			"required",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.UnknownVal(cty.DynamicPseudoType),
+			`Required variable not set: Unsuitable value for var.required set from outside of the configuration: required variable may not be set to null.`,
+		},
+		{
+			"required",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"required",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// optional_default_string
+		{
+			"optional_default_string",
+			cty.NilVal,
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"optional_default_string",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"optional_default_string",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"optional_default_string",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_required
+		{
+			"constrained_string_required",
+			cty.NilVal,
+			cty.UnknownVal(cty.String),
+			`Required variable not set: The variable "constrained_string_required" is required, but is not set.`,
+		},
+		{
+			"constrained_string_required",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.UnknownVal(cty.String),
+			`Required variable not set: Unsuitable value for var.constrained_string_required set from outside of the configuration: required variable may not be set to null.`,
+		},
+		{
+			"constrained_string_required",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_required",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_optional_default_string
+		{
+			"constrained_string_optional_default_string",
+			cty.NilVal,
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"constrained_string_optional_default_string",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.StringVal("hello"), // the declared default value
+			``,
+		},
+		{
+			"constrained_string_optional_default_string",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_optional_default_string",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+
+		// constrained_string_optional_default_bool
+		{
+			"constrained_string_optional_default_bool",
+			cty.NilVal,
+			cty.StringVal("true"), // the declared default value, automatically converted to match type constraint
+			``,
+		},
+		{
+			"constrained_string_optional_default_bool",
+			cty.NullVal(cty.DynamicPseudoType),
+			cty.StringVal("true"), // the declared default value, automatically converted to match type constraint
+			``,
+		},
+		{
+			"constrained_string_optional_default_bool",
+			cty.StringVal("ahoy"),
+			cty.StringVal("ahoy"),
+			``,
+		},
+		{
+			"constrained_string_optional_default_bool",
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+			``,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s %#v", test.varName, test.given), func(t *testing.T) {
+			varAddr := addrs.InputVariable{Name: test.varName}.Absolute(addrs.RootModuleInstance)
+			varCfg := variableConfigs[test.varName]
+			if varCfg == nil {
+				t.Fatalf("invalid variable name %q", test.varName)
+			}
+
+			t.Logf(
+				"test case\nvariable:    %s\nconstraint:  %#v\ndefault:     %#v\nnullable:    %#v\ngiven value: %#v",
+				varAddr,
+				varCfg.Type,
+				varCfg.Default,
+				varCfg.Nullable,
+				test.given,
+			)
+
+			rawVal := &InputValue{
+				Value:      test.given,
+				SourceType: ValueFromCaller,
+			}
+
+			got, diags := prepareFinalInputVariableValue(
+				varAddr, rawVal, varCfg,
+			)
+
+			if test.wantErr != "" {
+				if !diags.HasErrors() {
+					t.Errorf("unexpected success\nwant error: %s", test.wantErr)
+				} else if got, want := diags.Err().Error(), test.wantErr; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+			} else {
+				if diags.HasErrors() {
+					t.Errorf("unexpected error\ngot: %s", diags.Err().Error())
+				}
+			}
+
+			// NOTE: should still have returned some reasonable value even if there was an error
+			if !test.want.RawEquals(got) {
+				t.Fatalf("wrong result\ngot:  %#v\nwant: %#v", got, test.want)
+			}
+		})
+	}
+
+	t.Run("SourceType error message variants", func(t *testing.T) {
+		tests := []struct {
+			SourceType  ValueSourceType
+			SourceRange tfdiags.SourceRange
+			WantTypeErr string
+			WantNullErr string
+		}{
+			{
+				ValueFromUnknown,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set from outside of the configuration: string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set from outside of the configuration: required variable may not be set to null.`,
+			},
+			{
+				ValueFromConfig,
+				tfdiags.SourceRange{
+					Filename: "example.tf",
+					Start:    tfdiags.SourcePos(hcl.InitialPos),
+					End:      tfdiags.SourcePos(hcl.InitialPos),
+				},
+				`Invalid value for input variable: The given value is not suitable for var.constrained_string_required declared at main.tf:32,3-41: string required.`,
+				`Required variable not set: The given value is not suitable for var.constrained_string_required defined at main.tf:32,3-41: required variable may not be set to null.`,
+			},
+			{
+				ValueFromAutoFile,
+				tfdiags.SourceRange{
+					Filename: "example.auto.tfvars",
+					Start:    tfdiags.SourcePos(hcl.InitialPos),
+					End:      tfdiags.SourcePos(hcl.InitialPos),
+				},
+				`Invalid value for input variable: The given value is not suitable for var.constrained_string_required declared at main.tf:32,3-41: string required.`,
+				`Required variable not set: The given value is not suitable for var.constrained_string_required defined at main.tf:32,3-41: required variable may not be set to null.`,
+			},
+			{
+				ValueFromNamedFile,
+				tfdiags.SourceRange{
+					Filename: "example.tfvars",
+					Start:    tfdiags.SourcePos(hcl.InitialPos),
+					End:      tfdiags.SourcePos(hcl.InitialPos),
+				},
+				`Invalid value for input variable: The given value is not suitable for var.constrained_string_required declared at main.tf:32,3-41: string required.`,
+				`Required variable not set: The given value is not suitable for var.constrained_string_required defined at main.tf:32,3-41: required variable may not be set to null.`,
+			},
+			{
+				ValueFromCLIArg,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set using -var="constrained_string_required=...": string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set using -var="constrained_string_required=...": required variable may not be set to null.`,
+			},
+			{
+				ValueFromEnvVar,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set using the TF_VAR_constrained_string_required environment variable: string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set using the TF_VAR_constrained_string_required environment variable: required variable may not be set to null.`,
+			},
+			{
+				ValueFromInput,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set using an interactive prompt: string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set using an interactive prompt: required variable may not be set to null.`,
+			},
+			{
+				// NOTE: This isn't actually a realistic case for this particular
+				// function, because if we have a value coming from a plan then
+				// we must be in the apply step, and we shouldn't be able to
+				// get past the plan step if we have invalid variable values,
+				// and during planning we'll always have other source types.
+				ValueFromPlan,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set from outside of the configuration: string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set from outside of the configuration: required variable may not be set to null.`,
+			},
+			{
+				ValueFromCaller,
+				tfdiags.SourceRange{},
+				`Invalid value for input variable: Unsuitable value for var.constrained_string_required set from outside of the configuration: string required.`,
+				`Required variable not set: Unsuitable value for var.constrained_string_required set from outside of the configuration: required variable may not be set to null.`,
+			},
+		}
+
+		for _, test := range tests {
+			t.Run(fmt.Sprintf("%s %s", test.SourceType, test.SourceRange.StartString()), func(t *testing.T) {
+				varAddr := addrs.InputVariable{Name: "constrained_string_required"}.Absolute(addrs.RootModuleInstance)
+				varCfg := variableConfigs[varAddr.Variable.Name]
+				t.Run("type error", func(t *testing.T) {
+					rawVal := &InputValue{
+						Value:       cty.EmptyObjectVal,
+						SourceType:  test.SourceType,
+						SourceRange: test.SourceRange,
+					}
+
+					_, diags := prepareFinalInputVariableValue(
+						varAddr, rawVal, varCfg,
+					)
+					if !diags.HasErrors() {
+						t.Fatalf("unexpected success; want error")
+					}
+
+					if got, want := diags.Err().Error(), test.WantTypeErr; got != want {
+						t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+					}
+				})
+				t.Run("null error", func(t *testing.T) {
+					rawVal := &InputValue{
+						Value:       cty.NullVal(cty.DynamicPseudoType),
+						SourceType:  test.SourceType,
+						SourceRange: test.SourceRange,
+					}
+
+					_, diags := prepareFinalInputVariableValue(
+						varAddr, rawVal, varCfg,
+					)
+					if !diags.HasErrors() {
+						t.Fatalf("unexpected success; want error")
+					}
+
+					if got, want := diags.Err().Error(), test.WantNullErr; got != want {
+						t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+					}
+				})
+			})
+		}
+	})
+}

internal/terraform/evaluate.go

@@ -10,7 +10,6 @@ import (
 	"github.com/agext/levenshtein"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/zclconf/go-cty/cty"
-	"github.com/zclconf/go-cty/cty/convert"
 
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/configs"
@@ -248,7 +247,7 @@ func (d *evaluationStateData) GetInputVariable(addr addrs.InputVariable, rng tfd
 	// This is important because otherwise the validation walk will tend to be
 	// overly strict, requiring expressions throughout the configuration to
 	// be complicated to accommodate all possible inputs, whereas returning
-	// known here allows for simpler patterns like using input values as
+	// unknown here allows for simpler patterns like using input values as
 	// guards to broadly enable/disable resources, avoid processing things
 	// that are disabled, etc. Terraform's static validation leans towards
 	// being liberal in what it accepts because the subsequent plan walk has
@@ -267,28 +266,27 @@ func (d *evaluationStateData) GetInputVariable(addr addrs.InputVariable, rng tfd
 		return cty.UnknownVal(config.Type), diags
 	}
 
+	// d.Evaluator.VariableValues should always contain valid "final values"
+	// for variables, which is to say that they have already had type
+	// conversions, validations, and default value handling applied to them.
+	// Those are the responsibility of the graph notes representing the
+	// variable declarations. Therefore here we just trust that we already
+	// have a correct value.
+
 	val, isSet := vals[addr.Name]
-	switch {
+	if !isSet {
-	case !isSet:
+		// We should not be able to get here without having a valid value
-		// The config loader will ensure there is a default if the value is not
+		// for every variable, so this always indicates a bug in either
-		// set at all.
+		// the graph builder (not including all the needed nodes) or in
-		val = config.Default
+		// the graph nodes representing variables.
-
-	case val.IsNull() && !config.Nullable && config.Default != cty.NilVal:
-		// If nullable=false a null value will use the configured default.
-		val = config.Default
-	}
-
-	var err error
-	val, err = convert.Convert(val, config.ConstraintType)
-	if err != nil {
-		// We should never get here because this problem should've been caught
-		// during earlier validation, but we'll do something reasonable anyway.
 		diags = diags.Append(&hcl.Diagnostic{
 			Severity: hcl.DiagError,
-			Summary:  `Incorrect variable type`,
+			Summary:  `Reference to unresolved input variable`,
-			Detail:   fmt.Sprintf(`The resolved value of variable %q is not appropriate: %s.`, addr.Name, err),
+			Detail: fmt.Sprintf(
-			Subject:  &config.DeclRange,
+				`The final value for %s is missing in Terraform's evaluation context. This is a bug in Terraform; please report it!`,
+				addr.Absolute(d.ModulePath),
+			),
+			Subject: rng.ToHCL().Ptr(),
 		})
 		val = cty.UnknownVal(config.Type)
 	}

internal/terraform/graph_builder_apply.go

@@ -26,6 +26,11 @@ type ApplyGraphBuilder struct {
 	// State is the current state
 	State *states.State
 
+	// RootVariableValues are the root module input variables captured as
+	// part of the plan object, which we must reproduce in the apply step
+	// to get a consistent result.
+	RootVariableValues InputValues
+
 	// Plugins is a library of the plug-in components (providers and
 	// provisioners) available for use.
 	Plugins *contextPlugins
@@ -88,7 +93,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
 		},
 
 		// Add dynamic values
-		&RootVariableTransformer{Config: b.Config},
+		&RootVariableTransformer{Config: b.Config, RawValues: b.RootVariableValues},
 		&ModuleVariableTransformer{Config: b.Config},
 		&LocalTransformer{Config: b.Config},
 		&OutputTransformer{Config: b.Config, Changes: b.Changes},