workdir: Start of a new package for working directory state management

Thus far our various interactions with the bits of state we keep
associated with a working directory have all been implemented directly
inside the "command" package -- often in the huge command.Meta type -- and
not managed collectively via a single component.

There's too many little codepaths reading and writing from the working
directory and data directory to refactor it all in one step, but this is
an attempt at a first step towards a future where everything that reads
and writes from the current working directory would do so via an object
that encapsulates the implementation details and offers a high-level API
to read and write all of these session-persistent settings.

The design here continues our gradual path towards using a dependency
injection style where "package main" is solely responsible for directly
interacting with the OS command line, the OS environment, the OS working
directory, the stdio streams, and the CLI configuration, and then
communicating the resulting information to the rest of Terraform by wiring
together objects. It seems likely that eventually we'll have enough wiring
code in package main to justify a more explicit organization of that code,
but for this commit the new "workdir.Dir" object is just wired directly in
place of its predecessors, without any significant change of code
organization at that top layer.

This first commit focuses on the main files and directories we use to
find provider plugins, because a subsequent commit will lightly reorganize
the separation of concerns for plugin launching with a similar goal of
collecting all of the relevant logic together into one spot.

commands.go

@@ -77,12 +77,12 @@ func initCommands(
 		configDir = "" // No config dir available (e.g. looking up a home directory failed)
 	}
 
-	dataDir := os.Getenv("TF_DATA_DIR")
+	wd := WorkingDir(originalWorkingDir, os.Getenv("TF_DATA_DIR"))
 
 	meta := command.Meta{
-		OriginalWorkingDir: originalWorkingDir,
-		Streams:            streams,
-		View:               views.NewView(streams).SetRunningInAutomation(inAutomation),
+		WorkingDir: wd,
+		Streams:    streams,
+		View:       views.NewView(streams).SetRunningInAutomation(inAutomation),
 
 		Color:            true,
 		GlobalPluginDirs: globalPluginDirs(),
@@ -94,7 +94,6 @@ func initCommands(
 		RunningInAutomation: inAutomation,
 		CLIConfigDir:        configDir,
 		PluginCacheDir:      config.PluginCacheDir,
-		OverrideDataDir:     dataDir,
 
 		ShutdownCh: makeShutdownCh(),
 

internal/command/command_test.go

@@ -24,6 +24,7 @@ import (
 	backendInit "github.com/hashicorp/terraform/internal/backend/init"
 	backendLocal "github.com/hashicorp/terraform/internal/backend/local"
 	"github.com/hashicorp/terraform/internal/command/views"
+	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs"
 	"github.com/hashicorp/terraform/internal/configs/configload"
 	"github.com/hashicorp/terraform/internal/configs/configschema"
@@ -108,6 +109,65 @@ func tempDir(t *testing.T) string {
 	return dir
 }
 
+// tempWorkingDir constructs a workdir.Dir object referring to a newly-created
+// temporary directory, and returns that object along with a cleanup function
+// to call once the calling test is complete.
+//
+// Although workdir.Dir is built to support arbitrary base directories, the
+// not-yet-migrated behaviors in command.Meta tend to expect the root module
+// directory to be the real process working directory, and so if you intend
+// to use the result inside a command.Meta object you must use a pattern
+// similar to the following when initializing your test:
+//
+//     wd, cleanup := tempWorkingDir(t)
+//     defer cleanup()
+//     defer testChdir(t, wd.RootModuleDir())()
+//
+// Note that testChdir modifies global state for the test process, and so a
+// test using this pattern must never call t.Parallel().
+func tempWorkingDir(t *testing.T) (*workdir.Dir, func() error) {
+	t.Helper()
+
+	dirPath, err := os.MkdirTemp("", "tf-command-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := func() error {
+		return os.RemoveAll(dirPath)
+	}
+	t.Logf("temporary directory %s", dirPath)
+
+	return workdir.NewDir(dirPath), done
+}
+
+// tempWorkingDirFixture is like tempWorkingDir but it also copies the content
+// from a fixture directory into the temporary directory before returning it.
+//
+// The same caveats about working directory apply as for testWorkingDir. See
+// the testWorkingDir commentary for an example of how to use this function
+// along with testChdir to meet the expectations of command.Meta legacy
+// functionality.
+func tempWorkingDirFixture(t *testing.T, fixtureName string) (*workdir.Dir, func() error) {
+	t.Helper()
+
+	dirPath, err := os.MkdirTemp("", "tf-command-test-"+fixtureName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := func() error {
+		return os.RemoveAll(dirPath)
+	}
+	t.Logf("temporary directory %s with fixture %q", dirPath, fixtureName)
+
+	fixturePath := testFixturePath(fixtureName)
+	testCopyDir(t, fixturePath, dirPath)
+	// NOTE: Unfortunately because testCopyDir immediately aborts the test
+	// on failure, a failure to copy will prevent us from cleaning up the
+	// temporary directory. Oh well. :(
+
+	return workdir.NewDir(dirPath), done
+}
+
 func testFixturePath(name string) string {
 	return filepath.Join(fixtureDir, name)
 }
@@ -853,8 +913,10 @@ func testLockState(sourceDir, path string) (func(), error) {
 }
 
 // testCopyDir recursively copies a directory tree, attempting to preserve
-// permissions. Source directory must exist, destination directory must *not*
-// exist. Symlinks are ignored and skipped.
+// permissions. Source directory must exist, destination directory may exist
+// but will be created if not; it should typically be a temporary directory,
+// and thus already created using os.MkdirTemp or similar.
+// Symlinks are ignored and skipped.
 func testCopyDir(t *testing.T, src, dst string) {
 	t.Helper()
 
@@ -873,9 +935,6 @@ func testCopyDir(t *testing.T, src, dst string) {
 	if err != nil && !os.IsNotExist(err) {
 		t.Fatal(err)
 	}
-	if err == nil {
-		t.Fatal("destination already exists")
-	}
 
 	err = os.MkdirAll(dst, si.Mode())
 	if err != nil {

internal/command/get_test.go

@@ -1,7 +1,6 @@
 package command
 
 import (
-	"os"
 	"strings"
 	"testing"
 
@@ -9,17 +8,16 @@ import (
 )
 
 func TestGet(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("get"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "get")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 
@@ -35,12 +33,16 @@ func TestGet(t *testing.T) {
 }
 
 func TestGet_multipleArgs(t *testing.T) {
-	ui := new(cli.MockUi)
+	wd, cleanup := tempWorkingDir(t)
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
+
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 
@@ -54,17 +56,16 @@ func TestGet_multipleArgs(t *testing.T) {
 }
 
 func TestGet_update(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("get"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "get")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
-	ui := new(cli.MockUi)
+	ui := cli.NewMockUi()
 	c := &GetCommand{
 		Meta: Meta{
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
-			dataDir:          tempDir(t),
+			WorkingDir:       wd,
 		},
 	}
 

internal/command/meta.go

@@ -16,6 +16,9 @@ import (
 
 	plugin "github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/terraform-svchost/disco"
+	"github.com/mitchellh/cli"
+	"github.com/mitchellh/colorstring"
+
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/backend"
 	"github.com/hashicorp/terraform/internal/backend/local"
@@ -23,17 +26,15 @@ import (
 	"github.com/hashicorp/terraform/internal/command/format"
 	"github.com/hashicorp/terraform/internal/command/views"
 	"github.com/hashicorp/terraform/internal/command/webbrowser"
+	"github.com/hashicorp/terraform/internal/command/workdir"
 	"github.com/hashicorp/terraform/internal/configs/configload"
 	"github.com/hashicorp/terraform/internal/getproviders"
+	legacy "github.com/hashicorp/terraform/internal/legacy/terraform"
 	"github.com/hashicorp/terraform/internal/providers"
 	"github.com/hashicorp/terraform/internal/provisioners"
 	"github.com/hashicorp/terraform/internal/terminal"
 	"github.com/hashicorp/terraform/internal/terraform"
 	"github.com/hashicorp/terraform/internal/tfdiags"
-	"github.com/mitchellh/cli"
-	"github.com/mitchellh/colorstring"
-
-	legacy "github.com/hashicorp/terraform/internal/legacy/terraform"
 )
 
 // Meta are the meta-options that are available on all or most commands.
@@ -42,16 +43,19 @@ type Meta struct {
 	// command with a Meta field. These are expected to be set externally
 	// (not from within the command itself).
 
-	// OriginalWorkingDir, if set, is the actual working directory where
-	// Terraform was run from. This might not be the _actual_ current working
-	// directory, because users can add the -chdir=... option to the beginning
-	// of their command line to ask Terraform to switch.
+	// WorkingDir is an object representing the "working directory" where we're
+	// running commands. In the normal case this literally refers to the
+	// working directory of the Terraform process, though this can take on
+	// a more symbolic meaning when the user has overridden default behavior
+	// to specify a different working directory or to override the special
+	// data directory where we'll persist settings that must survive between
+	// consecutive commands.
 	//
-	// Most things should just use the current working directory in order to
-	// respect the user's override, but we retain this for exceptional
-	// situations where we need to refer back to the original working directory
-	// for some reason.
-	OriginalWorkingDir string
+	// We're currently gradually migrating the various bits of state that
+	// must persist between consecutive commands in a session to be encapsulated
+	// in here, but we're not there yet and so there are also some methods on
+	// Meta which directly read and modify paths inside the data directory.
+	WorkingDir *workdir.Dir
 
 	// Streams tracks the raw Stdout, Stderr, and Stdin handles along with
 	// some basic metadata about them, such as whether each is connected to
@@ -102,11 +106,6 @@ type Meta struct {
 	// provider version can be obtained.
 	ProviderSource getproviders.Source
 
-	// OverrideDataDir, if non-empty, overrides the return value of the
-	// DataDir method for situations where the local .terraform/ directory
-	// is not suitable, e.g. because of a read-only filesystem.
-	OverrideDataDir string
-
 	// BrowserLauncher is used by commands that need to open a URL in a
 	// web browser.
 	BrowserLauncher webbrowser.Launcher
@@ -135,10 +134,6 @@ type Meta struct {
 	// Protected: commands can set these
 	//----------------------------------------------------------
 
-	// Modify the data directory location. This should be accessed through the
-	// DataDir method.
-	dataDir string
-
 	// pluginPath is a user defined set of directories to look for plugins.
 	// This is set during init with the `-plugin-dir` flag, saved to a file in
 	// the data directory.
@@ -265,13 +260,25 @@ func (m *Meta) Colorize() *colorstring.Colorize {
 	}
 }
 
+// fixupMissingWorkingDir is a compensation for various existing tests which
+// directly construct incomplete "Meta" objects. Specifically, it deals with
+// a test that omits a WorkingDir value by constructing one just-in-time.
+//
+// We shouldn't ever rely on this in any real codepath, because it doesn't
+// take into account the various ways users can override our default
+// directory selection behaviors.
+func (m *Meta) fixupMissingWorkingDir() {
+	if m.WorkingDir == nil {
+		log.Printf("[WARN] This 'Meta' object is missing its WorkingDir, so we're creating a default one suitable only for tests")
+		m.WorkingDir = workdir.NewDir(".")
+	}
+}
+
 // DataDir returns the directory where local data will be stored.
 // Defaults to DefaultDataDir in the current working directory.
 func (m *Meta) DataDir() string {
-	if m.OverrideDataDir != "" {
-		return m.OverrideDataDir
-	}
-	return DefaultDataDir
+	m.fixupMissingWorkingDir()
+	return m.WorkingDir.DataDir()
 }
 
 const (
@@ -499,7 +506,7 @@ func (m *Meta) contextOpts() (*terraform.ContextOpts, error) {
 
 	opts.Meta = &terraform.ContextMeta{
 		Env:                workspace,
-		OriginalWorkingDir: m.OriginalWorkingDir,
+		OriginalWorkingDir: m.WorkingDir.OriginalWorkingDir(),
 	}
 
 	return &opts, nil

internal/command/meta_backend_test.go

@@ -1855,17 +1855,19 @@ func TestMetaBackend_configToExtra(t *testing.T) {
 
 // no config; return inmem backend stored in state
 func TestBackendFromState(t *testing.T) {
-	td := tempDir(t)
-	testCopyDir(t, testFixturePath("backend-from-state"), td)
-	defer os.RemoveAll(td)
-	defer testChdir(t, td)()
+	wd, cleanup := tempWorkingDirFixture(t, "backend-from-state")
+	defer cleanup()
+	defer testChdir(t, wd.RootModuleDir())()
 
 	// Setup the meta
 	m := testMetaBackend(t, nil)
+	m.WorkingDir = wd
 	// terraform caches a small "state" file that stores the backend config.
 	// This test must override m.dataDir so it loads the "terraform.tfstate" file in the
-	// test directory as the backend config cache
-	m.OverrideDataDir = td
+	// test directory as the backend config cache. This fixture is really a
+	// fixture for the data dir rather than the module dir, so we'll override
+	// them to match just for this test.
+	wd.OverrideDataDir(".")
 
 	stateBackend, diags := m.backendFromState()
 	if diags.HasErrors() {

internal/command/meta_config.go

@@ -27,27 +27,8 @@ import (
 // paths used to load configuration, because we want to prefer recording
 // relative paths in source code references within the configuration.
 func (m *Meta) normalizePath(path string) string {
-	var err error
-
-	// First we will make it absolute so that we have a consistent place
-	// to start.
-	path, err = filepath.Abs(path)
-	if err != nil {
-		// We'll just accept what we were given, then.
-		return path
-	}
-
-	cwd, err := os.Getwd()
-	if err != nil || !filepath.IsAbs(cwd) {
-		return path
-	}
-
-	ret, err := filepath.Rel(cwd, path)
-	if err != nil {
-		return path
-	}
-
-	return ret
+	m.fixupMissingWorkingDir()
+	return m.WorkingDir.NormalizePath(path)
 }
 
 // loadConfig reads a configuration from the given directory, which should

internal/command/meta_providers.go

@@ -6,7 +6,6 @@ import (
 	"log"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
 	"github.com/hashicorp/go-multierror"
@@ -109,7 +108,8 @@ func (m *Meta) providerCustomLocalDirectorySource(dirs []string) getproviders.So
 // Only one object returned from this method should be live at any time,
 // because objects inside contain caches that must be maintained properly.
 func (m *Meta) providerLocalCacheDir() *providercache.Dir {
-	dir := filepath.Join(m.DataDir(), "providers")
+	m.fixupMissingWorkingDir()
+	dir := m.WorkingDir.ProviderLocalCacheDir()
 	return providercache.NewDir(dir)
 }
 

internal/command/plugins.go

@@ -1,11 +1,8 @@
 package command
 
 import (
-	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
-	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
@@ -36,46 +33,21 @@ func (m *Meta) storePluginPath(pluginPath []string) error {
 		return nil
 	}
 
-	path := filepath.Join(m.DataDir(), PluginPathFile)
+	m.fixupMissingWorkingDir()
 
 	// remove the plugin dir record if the path was set to an empty string
 	if len(pluginPath) == 1 && (pluginPath[0] == "") {
-		err := os.Remove(path)
-		if !os.IsNotExist(err) {
-			return err
-		}
-		return nil
+		return m.WorkingDir.SetForcedPluginDirs(nil)
 	}
 
-	js, err := json.MarshalIndent(pluginPath, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	// if this fails, so will WriteFile
-	os.MkdirAll(m.DataDir(), 0755)
-
-	return ioutil.WriteFile(path, js, 0644)
+	return m.WorkingDir.SetForcedPluginDirs(pluginPath)
 }
 
 // Load the user-defined plugin search path into Meta.pluginPath if the file
 // exists.
 func (m *Meta) loadPluginPath() ([]string, error) {
-	js, err := ioutil.ReadFile(filepath.Join(m.DataDir(), PluginPathFile))
-	if os.IsNotExist(err) {
-		return nil, nil
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	var pluginPath []string
-	if err := json.Unmarshal(js, &pluginPath); err != nil {
-		return nil, err
-	}
-
-	return pluginPath, nil
+	m.fixupMissingWorkingDir()
+	return m.WorkingDir.ForcedPluginDirs()
 }
 
 // the default location for automatically installed plugins

internal/command/workdir/dir.go

@@ -0,0 +1,149 @@
+package workdir
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Dir represents a single Terraform working directory.
+//
+// "Working directory" is unfortunately a slight misnomer, because non-default
+// options can potentially stretch the definition such that multiple working
+// directories end up appearing to share a data directory, or other similar
+// anomolies, but we continue to use this terminology both for historical
+// reasons and because it reflects the common case without any special
+// overrides.
+//
+// The naming convention for methods on this type is that methods whose names
+// begin with "Override" affect only characteristics of the particular object
+// they're called on, changing where it looks for data, while methods whose
+// names begin with "Set" will write settings to disk such that other instances
+// referring to the same directories will also see them. Given that, the
+// "Override" methods should be used only during the initialization steps
+// for a Dir object, typically only inside "package main", so that all
+// subsequent work elsewhere will access consistent locations on disk.
+//
+// We're gradually transitioning to using this type to manage working directory
+// settings, and so not everything in the working directory "data dir" is
+// encapsulated here yet, but hopefully we'll gradually migrate all of those
+// settings here over time. The working directory state not yet managed in here
+// is typically managed directly in the "command" package, either directly
+// inside commands or in methods of the giant command.Meta type.
+type Dir struct {
+	// mainDir is the path to the directory that we present as the
+	// "working directory" in the user model, which is typically the
+	// current working directory when running Terraform CLI, or the
+	// directory explicitly chosen by the user using the -chdir=...
+	// global option.
+	mainDir string
+
+	// originalDir is the path to the working directory that was
+	// selected when creating the Terraform CLI process, regardless of
+	// -chdir=... being set. This is only for very limited purposes
+	// related to backward compatibility; most functionality should
+	// use mainDir instead.
+	originalDir string
+
+	// dataDir is the path to the directory where we will store our
+	// working directory settings and artifacts. This is typically a
+	// directory named ".terraform" within mainDir, but users may
+	// override it.
+	dataDir string
+}
+
+// NewDir constructs a new working directory, anchored at the given path.
+//
+// In normal use, mainPath should be "." to reflect the current working
+// directory, with "package main" having switched the process's current
+// working directory if necessary prior to calling this function. However,
+// unusual situations in tests may set mainPath to a temporary directory, or
+// similar.
+//
+// WARNING: Although the logic in this package is intended to work regardless
+// of whether mainPath is actually the current working directory, we're
+// currently in a transitional state where this package shares responsibility
+// for the working directory with various command.Meta methods, and those
+// often assume that the main path of the working directory will always be
+// ".". If you're writing test code that spans across both areas of
+// responsibility then you must ensure that the test temporarily changes the
+// test process's working directory to the directory returned by RootModuleDir
+// before using the result inside a command.Meta.
+func NewDir(mainPath string) *Dir {
+	mainPath = filepath.Clean(mainPath)
+	return &Dir{
+		mainDir:     mainPath,
+		originalDir: mainPath,
+		dataDir:     filepath.Join(mainPath, ".terraform"),
+	}
+}
+
+// OverrideOriginalWorkingDir records a different path as the
+// "original working directory" for the reciever.
+//
+// Use this only to record the original working directory when Terraform is run
+// with the -chdir=... global option. In that case, the directory given in
+// -chdir=... is the "main path" to pass in to NewDir, while the original
+// working directory should be sent to this method.
+func (d *Dir) OverrideOriginalWorkingDir(originalPath string) {
+	d.originalDir = filepath.Clean(originalPath)
+}
+
+// OverrideDataDir chooses a specific alternative directory to read and write
+// the persistent working directory settings.
+//
+// "package main" can call this if it detects that the user has overridden
+// the default location by setting the relevant environment variable. Don't
+// call this when that environment variable isn't set, in order to preserve
+// the default setting of a dot-prefixed directory directly inside the main
+// working directory.
+func (d *Dir) OverrideDataDir(dataDir string) {
+	d.dataDir = filepath.Clean(dataDir)
+}
+
+// RootModuleDir returns the directory where we expect to find the root module
+// configuration for this working directory.
+func (d *Dir) RootModuleDir() string {
+	// The root module configuration is just directly inside the main directory.
+	return d.mainDir
+}
+
+// OriginalWorkingDir returns the true, operating-system-originated working
+// directory that the current Terraform process was launched from.
+//
+// This is usually the same as the main working directory, but differs in the
+// special case where the user ran Terraform with the global -chdir=...
+// option. This is here only for a few backward compatibility affordances
+// from before we had the -chdir=... option, so should typically not be used
+// for anything new.
+func (d *Dir) OriginalWorkingDir() string {
+	return d.originalDir
+}
+
+// DataDir returns the base path where the reciever keeps all of the settings
+// and artifacts that must persist between consecutive commands in a single
+// session.
+//
+// This is exported only to allow the legacy behaviors in command.Meta to
+// continue accessing this directory directly. Over time we should replace
+// all of those direct accesses with methods on this type, and then remove
+// this method. Avoid using this method for new use-cases.
+func (d *Dir) DataDir() string {
+	return d.dataDir
+}
+
+// ensureDataDir creates the data directory and all of the necessary parent
+// directories that lead to it, if they don't already exist.
+//
+// For directories that already exist ensureDataDir will preserve their
+// permissions, while it'll create any new directories to be owned by the user
+// running Terraform, readable and writable by that user, and readable by
+// all other users, or some approximation of that on non-Unix platforms which
+// have a different permissions model.
+func (d *Dir) ensureDataDir() error {
+	err := os.MkdirAll(d.dataDir, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to prepare working directory: %w", err)
+	}
+	return nil
+}

internal/command/workdir/doc.go

@@ -0,0 +1,16 @@
+// Package workdir models the various local artifacts and state we keep inside
+// a Terraform "working directory".
+//
+// The working directory artifacts and settings are typically initialized or
+// modified by "terraform init", after which they persist for use by other
+// commands in the same directory, but are not visible to commands run in
+// other working directories or on other computers.
+//
+// Although "terraform init" is the main command which modifies a workdir,
+// other commands do sometimes make more focused modifications for settings
+// which can typically change multiple times during a session, such as the
+// currently-selected workspace name. Any command which modifies the working
+// directory settings must discard and reload any objects which derived from
+// those settings, because otherwise the existing objects will often continue
+// to follow the settings that were present when they were created.
+package workdir

internal/command/workdir/normalize_path.go

@@ -0,0 +1,52 @@
+package workdir
+
+import (
+	"path/filepath"
+)
+
+// NormalizePath attempts to transform the given path so that it's relative
+// to the working directory, which is our preferred way to present and store
+// paths to files and directories within a configuration so that they can
+// be portable to operations in other working directories.
+//
+// It isn't always possible to produce a relative path. For example, on Windows
+// the given path might be on a different volume (e.g. drive letter or network
+// share) than the working directory.
+//
+// Note that the result will be relative to the main directory of the receiver,
+// which should always be the actual process working directory in normal code,
+// but might be some other temporary working directory when in test code.
+// If you need to access the file or directory that the result refers to with
+// functions that aren't aware of our base directory, you can use something
+// like the following, which again should be needed only in test code which
+// might need to inspect the filesystem in order to make assertions:
+//
+//     filepath.Join(d.RootModuleDir(), normalizePathResult)
+//
+// The above is suitable only for situations where the given path is known
+// to be beneath the working directory, which is the typical situation for
+// temporary working directories created for automated tests.
+func (d *Dir) NormalizePath(given string) string {
+	// We need an absolute version of d.mainDir in order for our "Rel"
+	// result to be reliable.
+	absMain, err := filepath.Abs(d.mainDir)
+	if err != nil {
+		// Weird, but okay...
+		return filepath.Clean(given)
+	}
+
+	if !filepath.IsAbs(given) {
+		given = filepath.Join(absMain, given)
+	}
+
+	ret, err := filepath.Rel(absMain, given)
+	if err != nil {
+		// It's not always possible to find a relative path. For example,
+		// the given path might be on an entirely separate volume
+		// (e.g. drive letter or network share) on a Windows system, which
+		// always requires an absolute path.
+		return filepath.Clean(given)
+	}
+
+	return ret
+}

internal/command/workdir/plugin_dirs.go

@@ -0,0 +1,83 @@
+package workdir
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const PluginPathFilename = "plugin_path"
+
+// ProviderLocalCacheDir returns the directory we'll use as the
+// working-directory-specific local cache of providers.
+//
+// The provider installer's job is to make sure that all providers needed for
+// a particular working directory are available in this cache directory. No
+// other component may write here, and in particular a Dir object itself
+// never reads or writes into this directory, instead just delegating all of
+// that responsibility to other components.
+//
+// Typically, the caller will ultimately pass the result of this method either
+// directly or indirectly into providercache.NewDir, to get an object
+// responsible for managing the contents.
+func (d *Dir) ProviderLocalCacheDir() string {
+	return filepath.Join(d.dataDir, "providers")
+}
+
+// ForcedPluginDirs returns a list of directories to use to find plugins,
+// instead of the default locations.
+//
+// Returns an zero-length list and no error in the normal case where there
+// are no overridden search directories. If ForcedPluginDirs returns a
+// non-empty list with no errors then the result totally replaces the default
+// search directories.
+func (d *Dir) ForcedPluginDirs() ([]string, error) {
+	raw, err := ioutil.ReadFile(filepath.Join(d.dataDir, PluginPathFilename))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	var pluginPath []string
+	if err := json.Unmarshal(raw, &pluginPath); err != nil {
+		return nil, err
+	}
+	return pluginPath, nil
+}
+
+// SetForcedPluginDirs records an overridden list of directories to search
+// to find plugins, instead of the default locations. See ForcePluginDirs
+// for more information.
+//
+// Pass a zero-length list to deactivate forced plugin directories altogether,
+// thus allowing the working directory to return to using the default
+// search directories.
+func (d *Dir) SetForcedPluginDirs(dirs []string) error {
+
+	filePath := filepath.Join(d.dataDir, PluginPathFilename)
+	switch {
+	case len(dirs) == 0:
+		err := os.Remove(filePath)
+		if !os.IsNotExist(err) {
+			return err
+		}
+		return nil
+	default:
+		// We'll ignore errors from this one, because if we fail to create
+		// the directory then we'll fail to create the file below too,
+		// and that subsequent error will more directly reflect what we
+		// are trying to do here.
+		d.ensureDataDir()
+
+		raw, err := json.MarshalIndent(dirs, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		return ioutil.WriteFile(filePath, raw, 0644)
+	}
+}

internal/command/workdir/plugin_dirs_test.go

@@ -0,0 +1,60 @@
+package workdir
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestDirForcedPluginDirs(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "terraform-workdir-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	dir := NewDir(tmpDir)
+	// We'll use the default convention of a data dir nested inside the
+	// working directory, so we don't need to override anything on "dir".
+
+	want := []string(nil)
+	got, err := dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong initial settings\n%s", diff)
+	}
+
+	fakeDir1 := filepath.Join(tmpDir, "boop1")
+	fakeDir2 := filepath.Join(tmpDir, "boop2")
+	err = dir.SetForcedPluginDirs([]string{fakeDir1, fakeDir2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []string{fakeDir1, fakeDir2}
+	got, err = dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong updated settings\n%s", diff)
+	}
+
+	err = dir.SetForcedPluginDirs(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = nil
+	got, err = dir.ForcedPluginDirs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("wrong final settings, after reverting back to defaults\n%s", diff)
+	}
+}

working_dir.go

@@ -0,0 +1,12 @@
+package main
+
+import "github.com/hashicorp/terraform/internal/command/workdir"
+
+func WorkingDir(originalDir string, overrideDataDir string) *workdir.Dir {
+	ret := workdir.NewDir(".") // caller should already have used os.Chdir in "-chdir=..." mode
+	ret.OverrideOriginalWorkingDir(originalDir)
+	if overrideDataDir != "" {
+		ret.OverrideDataDir(overrideDataDir)
+	}
+	return ret
+}