Move module install functionality over to internal/initwd

backend/local/backend_apply_test.go

@@ -13,8 +13,8 @@ import (
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/providers"
 	"github.com/hashicorp/terraform/states"
 	"github.com/hashicorp/terraform/states/statemgr"
@@ -252,7 +252,7 @@ func (s failingState) WriteState(state *states.State) error {
 func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) {
 	t.Helper()
 
-	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
+	_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
 
 	return &backend.Operation{
 		Type:         backend.OperationTypeApply,

backend/local/backend_plan_test.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/plans"
 	"github.com/hashicorp/terraform/plans/planfile"
 	"github.com/hashicorp/terraform/states"
@@ -514,7 +514,7 @@ func TestLocal_planScaleOutNoDupeCount(t *testing.T) {
 func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) {
 	t.Helper()
 
-	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
+	_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
 
 	return &backend.Operation{
 		Type:         backend.OperationTypePlan,

backend/local/backend_refresh_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/hashicorp/terraform/providers"
 
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/zclconf/go-cty/cty"
 )
@@ -203,7 +203,7 @@ test_instance.foo:
 func testOperationRefresh(t *testing.T, configDir string) (*backend.Operation, func()) {
 	t.Helper()
 
-	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
+	_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
 
 	return &backend.Operation{
 		Type:         backend.OperationTypeRefresh,

backend/remote/backend_apply_test.go

@@ -12,7 +12,7 @@ import (
 	tfe "github.com/hashicorp/go-tfe"
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/plans/planfile"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
@@ -21,7 +21,7 @@ import (
 func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) {
 	t.Helper()
 
-	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
+	_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
 
 	return &backend.Operation{
 		ConfigDir:    configDir,

backend/remote/backend_plan_test.go

@@ -12,7 +12,7 @@ import (
 	tfe "github.com/hashicorp/go-tfe"
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/backend"
-	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/plans/planfile"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/mitchellh/cli"
@@ -21,7 +21,7 @@ import (
 func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) {
 	t.Helper()
 
-	_, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir)
+	_, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir)
 
 	return &backend.Operation{
 		ConfigDir:    configDir,

command/command_test.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/hashicorp/terraform/internal/initwd"
+	"github.com/hashicorp/terraform/registry"
 	"io"
 	"io/ioutil"
 	"log"
@@ -141,7 +143,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
 	t.Helper()
 
 	dir := filepath.Join(fixtureDir, name)
-
 	// FIXME: We're not dealing with the cleanup function here because
 	// this testModule function is used all over and so we don't want to
 	// change its interface at this late stage.
@@ -150,9 +151,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
 	// Test modules usually do not refer to remote sources, and for local
 	// sources only this ultimately just records all of the module paths
 	// in a JSON file so that we can load them below.
-	diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{})
+	inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
-	if diags.HasErrors() {
+	instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{})
-		t.Fatal(diags.Error())
+	if instDiags.HasErrors() {
+		t.Fatal(instDiags.Err())
 	}
 
 	config, snap, diags := loader.LoadConfigWithSnapshot(dir)

command/hook_module_install.go

@@ -4,17 +4,17 @@ import (
 	"fmt"
 
 	version "github.com/hashicorp/go-version"
-	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/mitchellh/cli"
 )
 
 type uiModuleInstallHooks struct {
-	configload.InstallHooksImpl
+	initwd.ModuleInstallHooksImpl
 	Ui             cli.Ui
 	ShowLocalPaths bool
 }
 
-var _ configload.InstallHooks = uiModuleInstallHooks{}
+var _ initwd.ModuleInstallHooks = uiModuleInstallHooks{}
 
 func (h uiModuleInstallHooks) Download(modulePath, packageAddr string, v *version.Version) {
 	if v != nil {

command/meta_config.go

@@ -6,12 +6,13 @@ import (
 	"path/filepath"
 	"sort"
 
-	"github.com/hashicorp/hcl2/hcl/hclsyntax"
-
 	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcl/hclsyntax"
 	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/hashicorp/terraform/internal/initwd"
+	"github.com/hashicorp/terraform/registry"
 	"github.com/hashicorp/terraform/terraform"
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/zclconf/go-cty/cty"
@@ -165,7 +166,7 @@ func (m *Meta) loadHCLFile(filename string) (hcl.Body, tfdiags.Diagnostics) {
 // can then be relayed to the end-user. The moduleUiInstallHooks type in
 // this package has a reasonable implementation for displaying notifications
 // via a provided cli.Ui.
-func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.InstallHooks) tfdiags.Diagnostics {
+func (m *Meta) installModules(rootDir string, upgrade bool, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 	rootDir = m.normalizePath(rootDir)
 
@@ -175,14 +176,9 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins
 		return diags
 	}
 
-	loader, err := m.initConfigLoader()
+	inst := m.moduleInstaller()
-	if err != nil {
+	moreDiags := inst.InstallModules(rootDir, upgrade, hooks)
-		diags = diags.Append(err)
+	diags = diags.Append(moreDiags)
-		return diags
-	}
-
-	hclDiags := loader.InstallModules(rootDir, upgrade, hooks)
-	diags = diags.Append(hclDiags)
 	return diags
 }
 
@@ -195,18 +191,11 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins
 // can then be relayed to the end-user. The moduleUiInstallHooks type in
 // this package has a reasonable implementation for displaying notifications
 // via a provided cli.Ui.
-func (m *Meta) initDirFromModule(targetDir string, addr string, hooks configload.InstallHooks) tfdiags.Diagnostics {
+func (m *Meta) initDirFromModule(targetDir string, addr string, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 	targetDir = m.normalizePath(targetDir)
-
+	moreDiags := initwd.DirFromModule(targetDir, m.modulesDir(), addr, m.registryClient(), hooks)
-	loader, err := m.initConfigLoader()
+	diags = diags.Append(moreDiags)
-	if err != nil {
-		diags = diags.Append(err)
-		return diags
-	}
-
-	hclDiags := loader.InitDirFromModule(targetDir, addr, hooks)
-	diags = diags.Append(hclDiags)
 	return diags
 }
 
@@ -327,6 +316,18 @@ func (m *Meta) initConfigLoader() (*configload.Loader, error) {
 	return m.configLoader, nil
 }
 
+// moduleInstaller instantiates and returns a module installer for use by
+// "terraform init" (directly or indirectly).
+func (m *Meta) moduleInstaller() *initwd.ModuleInstaller {
+	reg := m.registryClient()
+	return initwd.NewModuleInstaller(m.modulesDir(), reg)
+}
+
+// registryClient instantiates and returns a new Terraform Registry client.
+func (m *Meta) registryClient() *registry.Client {
+	return registry.NewClient(m.Services, nil)
+}
+
 // configValueFromCLI parses a configuration value that was provided in a
 // context in the CLI where only strings can be provided, such as on the
 // command line or in an environment variable, and returns the resulting

configs/configload/loader.go

@@ -71,6 +71,12 @@ func NewLoader(config *Config) (*Loader, error) {
 	return ret, nil
 }
 
+// ModulesDir returns the path to the directory where the loader will look for
+// the local cache of remote module packages.
+func (l *Loader) ModulesDir() string {
+	return l.modules.Dir
+}
+
 // Parser returns the underlying parser for this loader.
 //
 // This is useful for loading other sorts of files than the module directories

configs/configload/loader_install.go

@@ -1,523 +0,0 @@
-package configload
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	version "github.com/hashicorp/go-version"
-	"github.com/hashicorp/hcl2/hcl"
-	"github.com/hashicorp/terraform/configs"
-	"github.com/hashicorp/terraform/registry"
-	"github.com/hashicorp/terraform/registry/regsrc"
-)
-
-// InstallModules analyses the root module in the given directory and installs
-// all of its direct and transitive dependencies into the loader's modules
-// directory, which must already exist.
-//
-// Since InstallModules makes possibly-time-consuming calls to remote services,
-// a hook interface is supported to allow the caller to be notified when
-// each module is installed and, for remote modules, when downloading begins.
-// LoadConfig guarantees that two hook calls will not happen concurrently but
-// it does not guarantee any particular ordering of hook calls. This mechanism
-// is for UI feedback only and does not give the caller any control over the
-// process.
-//
-// If modules are already installed in the target directory, they will be
-// skipped unless their source address or version have changed or unless
-// the upgrade flag is set.
-//
-// InstallModules never deletes any directory, except in the case where it
-// needs to replace a directory that is already present with a newly-extracted
-// package.
-//
-// If the returned diagnostics contains errors then the module installation
-// may have wholly or partially completed. Modules must be loaded in order
-// to find their dependencies, so this function does many of the same checks
-// as LoadConfig as a side-effect.
-//
-// This function will panic if called on a loader that cannot install modules.
-// Use CanInstallModules to determine if a loader can install modules, or
-// refer to the documentation for that method for situations where module
-// installation capability is guaranteed.
-func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
-	if !l.CanInstallModules() {
-		panic(fmt.Errorf("InstallModules called on loader that cannot install modules"))
-	}
-
-	rootMod, diags := l.parser.LoadConfigDir(rootDir)
-	if rootMod == nil {
-		return diags
-	}
-
-	getter := reusingGetter{}
-	instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks, getter)
-	diags = append(diags, instDiags...)
-
-	return diags
-}
-
-func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks, getter reusingGetter) hcl.Diagnostics {
-	var diags hcl.Diagnostics
-
-	if hooks == nil {
-		// Use our no-op implementation as a placeholder
-		hooks = InstallHooksImpl{}
-	}
-
-	// Create a manifest record for the root module. This will be used if
-	// there are any relative-pathed modules in the root.
-	l.modules.manifest[""] = moduleRecord{
-		Key: "",
-		Dir: rootDir,
-	}
-
-	_, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
-		func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
-
-			key := manifestKey(req.Path)
-			instPath := l.packageInstallPath(req.Path)
-
-			log.Printf("[DEBUG] Module installer: begin %s", key)
-
-			// First we'll check if we need to upgrade/replace an existing
-			// installed module, and delete it out of the way if so.
-			replace := upgrade
-			if !replace {
-				record, recorded := l.modules.manifest[key]
-				switch {
-				case !recorded:
-					log.Printf("[TRACE] %s is not yet installed", key)
-					replace = true
-				case record.SourceAddr != req.SourceAddr:
-					log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
-					replace = true
-				case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
-					log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
-					replace = true
-				}
-			}
-
-			// If we _are_ planning to replace this module, then we'll remove
-			// it now so our installation code below won't conflict with any
-			// existing remnants.
-			if replace {
-				if _, recorded := l.modules.manifest[key]; recorded {
-					log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
-				}
-				delete(l.modules.manifest, key)
-				// Deleting a module invalidates all of its descendent modules too.
-				keyPrefix := key + "."
-				for subKey := range l.modules.manifest {
-					if strings.HasPrefix(subKey, keyPrefix) {
-						if _, recorded := l.modules.manifest[subKey]; recorded {
-							log.Printf("[TRACE] also discarding downstream %s", subKey)
-						}
-						delete(l.modules.manifest, subKey)
-					}
-				}
-			}
-
-			record, recorded := l.modules.manifest[key]
-			if !recorded {
-				// Clean up any stale cache directory that might be present.
-				// If this is a local (relative) source then the dir will
-				// not exist, but we'll ignore that.
-				log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
-				err := l.modules.FS.RemoveAll(instPath)
-				if err != nil && !os.IsNotExist(err) {
-					log.Printf("[TRACE] failed to remove %s: %s", key, err)
-					diags = append(diags, &hcl.Diagnostic{
-						Severity: hcl.DiagError,
-						Summary:  "Failed to remove local module cache",
-						Detail: fmt.Sprintf(
-							"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
-							instPath, err,
-						),
-						Subject: &req.CallRange,
-					})
-					return nil, nil, diags
-				}
-			} else {
-				// If this module is already recorded and its root directory
-				// exists then we will just load what's already there and
-				// keep our existing record.
-				info, err := l.modules.FS.Stat(record.Dir)
-				if err == nil && info.IsDir() {
-					mod, mDiags := l.parser.LoadConfigDir(record.Dir)
-					diags = append(diags, mDiags...)
-
-					log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
-					return mod, record.Version, diags
-				}
-			}
-
-			// If we get down here then it's finally time to actually install
-			// the module. There are some variants to this process depending
-			// on what type of module source address we have.
-			switch {
-
-			case isLocalSourceAddr(req.SourceAddr):
-				log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
-				mod, mDiags := l.installLocalModule(req, key, hooks)
-				diags = append(diags, mDiags...)
-				return mod, nil, diags
-
-			case isRegistrySourceAddr(req.SourceAddr):
-				addr, err := regsrc.ParseModuleSource(req.SourceAddr)
-				if err != nil {
-					// Should never happen because isRegistrySourceAddr already validated
-					panic(err)
-				}
-				log.Printf("[TRACE] %s is a registry module at %s", key, addr)
-
-				mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks, getter)
-				diags = append(diags, mDiags...)
-				return mod, v, diags
-
-			default:
-				log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
-
-				mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks, getter)
-				diags = append(diags, mDiags...)
-				return mod, nil, diags
-			}
-
-		},
-	))
-	diags = append(diags, cDiags...)
-
-	err := l.modules.writeModuleManifestSnapshot()
-	if err != nil {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Failed to update module manifest",
-			Detail:   fmt.Sprintf("Unable to write the module manifest file: %s", err),
-		})
-	}
-
-	return diags
-}
-
-// CanInstallModules returns true if InstallModules can be used with this
-// loader.
-//
-// Loaders created with NewLoader can always install modules. Loaders created
-// from plan files (where the configuration is embedded in the plan file itself)
-// cannot install modules, because the plan file is read-only.
-func (l *Loader) CanInstallModules() bool {
-	return l.modules.CanInstall
-}
-
-func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
-	var diags hcl.Diagnostics
-
-	parentKey := manifestKey(req.Parent.Path)
-	parentRecord, recorded := l.modules.manifest[parentKey]
-	if !recorded {
-		// This is indicative of a bug rather than a user-actionable error
-		panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
-	}
-
-	if len(req.VersionConstraint.Required) != 0 {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Invalid version constraint",
-			Detail:   "A version constraint cannot be applied to a module at a relative local path.",
-			Subject:  &req.VersionConstraint.DeclRange,
-		})
-	}
-
-	// For local sources we don't actually need to modify the
-	// filesystem at all because the parent already wrote
-	// the files we need, and so we just load up what's already here.
-	newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
-	log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
-	mod, mDiags := l.parser.LoadConfigDir(newDir)
-	if mod == nil {
-		// nil indicates missing or unreadable directory, so we'll
-		// discard the returned diags and return a more specific
-		// error message here.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Unreadable module directory",
-			Detail:   fmt.Sprintf("The directory %s could not be read.", newDir),
-			Subject:  &req.SourceAddrRange,
-		})
-	} else {
-		diags = append(diags, mDiags...)
-	}
-
-	// Note the local location in our manifest.
-	l.modules.manifest[key] = moduleRecord{
-		Key:        key,
-		Dir:        newDir,
-		SourceAddr: req.SourceAddr,
-	}
-	log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
-	hooks.Install(key, nil, newDir)
-
-	return mod, diags
-}
-
-func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks, getter reusingGetter) (*configs.Module, *version.Version, hcl.Diagnostics) {
-	var diags hcl.Diagnostics
-
-	hostname, err := addr.SvcHost()
-	if err != nil {
-		// If it looks like the user was trying to use punycode then we'll generate
-		// a specialized error for that case. We require the unicode form of
-		// hostname so that hostnames are always human-readable in configuration
-		// and punycode can't be used to hide a malicious module hostname.
-		if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Invalid module registry hostname",
-				Detail:   "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.",
-				Subject:  &req.SourceAddrRange,
-			})
-		} else {
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Invalid module registry hostname",
-				Detail:   "The hostname portion of this source address is not a valid hostname.",
-				Subject:  &req.SourceAddrRange,
-			})
-		}
-		return nil, nil, diags
-	}
-
-	reg := l.modules.Registry
-
-	log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
-	resp, err := reg.ModuleVersions(addr)
-	if err != nil {
-		if registry.IsModuleNotFound(err) {
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Module not found",
-				Detail:   fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname),
-				Subject:  &req.SourceAddrRange,
-			})
-		} else {
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Error accessing remote module registry",
-				Detail:   fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err),
-				Subject:  &req.SourceAddrRange,
-			})
-		}
-		return nil, nil, diags
-	}
-
-	// The response might contain information about dependencies to allow us
-	// to potentially optimize future requests, but we don't currently do that
-	// and so for now we'll just take the first item which is guaranteed to
-	// be the address we requested.
-	if len(resp.Modules) < 1 {
-		// Should never happen, but since this is a remote service that may
-		// be implemented by third-parties we will handle it gracefully.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Invalid response from remote module registry",
-			Detail:   fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname),
-			Subject:  &req.SourceAddrRange,
-		})
-		return nil, nil, diags
-	}
-
-	modMeta := resp.Modules[0]
-
-	var latestMatch *version.Version
-	var latestVersion *version.Version
-	for _, mv := range modMeta.Versions {
-		v, err := version.NewVersion(mv.Version)
-		if err != nil {
-			// Should never happen if the registry server is compliant with
-			// the protocol, but we'll warn if not to assist someone who
-			// might be developing a module registry server.
-			diags = append(diags, &hcl.Diagnostic{
-				Severity: hcl.DiagWarning,
-				Summary:  "Invalid response from remote module registry",
-				Detail:   fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version),
-				Subject:  &req.SourceAddrRange,
-			})
-			continue
-		}
-
-		// If we've found a pre-release version then we'll ignore it unless
-		// it was exactly requested.
-		if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() {
-			log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
-			continue
-		}
-
-		if latestVersion == nil || v.GreaterThan(latestVersion) {
-			latestVersion = v
-		}
-
-		if req.VersionConstraint.Required.Check(v) {
-			if latestMatch == nil || v.GreaterThan(latestMatch) {
-				latestMatch = v
-			}
-		}
-	}
-
-	if latestVersion == nil {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Module has no versions",
-			Detail:   fmt.Sprintf("The specified module does not have any available versions."),
-			Subject:  &req.SourceAddrRange,
-		})
-		return nil, nil, diags
-	}
-
-	if latestMatch == nil {
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Unresolvable module version constraint",
-			Detail:   fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion),
-			Subject:  &req.VersionConstraint.DeclRange,
-		})
-		return nil, nil, diags
-	}
-
-	// Report up to the caller that we're about to start downloading.
-	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
-	hooks.Download(key, packageAddr, latestMatch)
-
-	// If we manage to get down here then we've found a suitable version to
-	// install, so we need to ask the registry where we should download it from.
-	// The response to this is a go-getter-style address string.
-	dlAddr, err := reg.ModuleLocation(addr, latestMatch.String())
-	if err != nil {
-		log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Invalid response from remote module registry",
-			Detail:   fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch),
-			Subject:  &req.VersionConstraint.DeclRange,
-		})
-		return nil, nil, diags
-	}
-
-	log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
-
-	modDir, err := getter.getWithGoGetter(instPath, dlAddr)
-	if err != nil {
-		// Errors returned by go-getter have very inconsistent quality as
-		// end-user error messages, but for now we're accepting that because
-		// we have no way to recognize any specific errors to improve them
-		// and masking the error entirely would hide valuable diagnostic
-		// information from the user.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Failed to download module",
-			Detail:   fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err),
-			Subject:  &req.CallRange,
-		})
-		return nil, nil, diags
-	}
-
-	log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
-
-	if addr.RawSubmodule != "" {
-		// Append the user's requested subdirectory to any subdirectory that
-		// was implied by any of the nested layers we expanded within go-getter.
-		modDir = filepath.Join(modDir, addr.RawSubmodule)
-	}
-
-	log.Printf("[TRACE] %s should now be at %s", key, modDir)
-
-	// Finally we are ready to try actually loading the module.
-	mod, mDiags := l.parser.LoadConfigDir(modDir)
-	if mod == nil {
-		// nil indicates missing or unreadable directory, so we'll
-		// discard the returned diags and return a more specific
-		// error message here. For registry modules this actually
-		// indicates a bug in the code above, since it's not the
-		// user's responsibility to create the directory in this case.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Unreadable module directory",
-			Detail:   fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
-			Subject:  &req.CallRange,
-		})
-	} else {
-		diags = append(diags, mDiags...)
-	}
-
-	// Note the local location in our manifest.
-	l.modules.manifest[key] = moduleRecord{
-		Key:        key,
-		Version:    latestMatch,
-		Dir:        modDir,
-		SourceAddr: req.SourceAddr,
-	}
-	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
-	hooks.Install(key, latestMatch, modDir)
-
-	return mod, latestMatch, diags
-}
-
-func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks, getter reusingGetter) (*configs.Module, hcl.Diagnostics) {
-	var diags hcl.Diagnostics
-
-	// Report up to the caller that we're about to start downloading.
-	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
-	hooks.Download(key, packageAddr, nil)
-
-	modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr)
-	if err != nil {
-		// Errors returned by go-getter have very inconsistent quality as
-		// end-user error messages, but for now we're accepting that because
-		// we have no way to recognize any specific errors to improve them
-		// and masking the error entirely would hide valuable diagnostic
-		// information from the user.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Failed to download module",
-			Detail:   fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err),
-			Subject:  &req.SourceAddrRange,
-		})
-		return nil, diags
-	}
-
-	log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
-
-	mod, mDiags := l.parser.LoadConfigDir(modDir)
-	if mod == nil {
-		// nil indicates missing or unreadable directory, so we'll
-		// discard the returned diags and return a more specific
-		// error message here. For registry modules this actually
-		// indicates a bug in the code above, since it's not the
-		// user's responsibility to create the directory in this case.
-		diags = append(diags, &hcl.Diagnostic{
-			Severity: hcl.DiagError,
-			Summary:  "Unreadable module directory",
-			Detail:   fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
-			Subject:  &req.CallRange,
-		})
-	} else {
-		diags = append(diags, mDiags...)
-	}
-
-	// Note the local location in our manifest.
-	l.modules.manifest[key] = moduleRecord{
-		Key:        key,
-		Dir:        modDir,
-		SourceAddr: req.SourceAddr,
-	}
-	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
-	hooks.Install(key, nil, modDir)
-
-	return mod, diags
-}
-
-func (l *Loader) packageInstallPath(modulePath []string) string {
-	return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
-}

configs/configload/loader_install_hooks.go

@@ -1,34 +0,0 @@
-package configload
-
-import version "github.com/hashicorp/go-version"
-
-// InstallHooks is an interface used to provide notifications about the
-// installation process being orchestrated by InstallModules.
-//
-// This interface may have new methods added in future, so implementers should
-// embed InstallHooksImpl to get no-op implementations of any unimplemented
-// methods.
-type InstallHooks interface {
-	// Download is called for modules that are retrieved from a remote source
-	// before that download begins, to allow a caller to give feedback
-	// on progress through a possibly-long sequence of downloads.
-	Download(moduleAddr, packageAddr string, version *version.Version)
-
-	// Install is called for each module that is installed, even if it did
-	// not need to be downloaded from a remote source.
-	Install(moduleAddr string, version *version.Version, localPath string)
-}
-
-// InstallHooksImpl is a do-nothing implementation of InstallHooks that
-// can be embedded in another implementation struct to allow only partial
-// implementation of the interface.
-type InstallHooksImpl struct {
-}
-
-func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
-}
-
-func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
-}
-
-var _ InstallHooks = InstallHooksImpl{}

configs/configload/loader_load.go

@@ -24,6 +24,18 @@ func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) {
 		return nil, diags
 	}
 
+	// Refresh the manifest snapshot in case anything new has been installed
+	// since we last refreshed it.
+	err := l.modules.readModuleManifestSnapshot()
+	if err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to read module manifest",
+			Detail:   fmt.Sprintf("Terraform failed to read its manifest of locally-cached modules: %s.", err),
+		})
+		return nil, diags
+	}
+
 	cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad))
 	diags = append(diags, cDiags...)
 
@@ -39,7 +51,7 @@ func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module,
 	// do verify that the manifest and the configuration are in agreement
 	// so that we can prompt the user to run "terraform init" if not.
 
-	key := manifestKey(req.Path)
+	key := l.modules.manifest.ModuleKey(req.Path)
 	record, exists := l.modules.manifest[key]
 
 	if !exists {

configs/configload/loader_snapshot.go

@@ -10,10 +10,9 @@ import (
 
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl2/hcl"
-	"github.com/spf13/afero"
-
-	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/internal/modsdir"
+	"github.com/spf13/afero"
 )
 
 // LoadConfigWithSnapshot is a variant of LoadConfig that also simultaneously
@@ -83,7 +82,7 @@ type Snapshot struct {
 func NewEmptySnapshot() *Snapshot {
 	return &Snapshot{
 		Modules: map[string]*SnapshotModule{
-			manifestKey(addrs.RootModule): &SnapshotModule{
+			"": &SnapshotModule{
 				Files: map[string][]byte{},
 			},
 		},
@@ -111,11 +110,11 @@ type SnapshotModule struct {
 
 // moduleManifest constructs a module manifest based on the contents of
 // the receiving snapshot.
-func (s *Snapshot) moduleManifest() moduleManifest {
+func (s *Snapshot) moduleManifest() modsdir.Manifest {
-	ret := make(moduleManifest)
+	ret := make(modsdir.Manifest)
 
 	for k, modSnap := range s.Modules {
-		ret[k] = moduleRecord{
+		ret[k] = modsdir.Record{
 			Key:        k,
 			Dir:        modSnap.Dir,
 			SourceAddr: modSnap.SourceAddr,
@@ -137,7 +136,7 @@ func (l *Loader) makeModuleWalkerSnapshot(snap *Snapshot) configs.ModuleWalker {
 				return mod, v, diags
 			}
 
-			key := manifestKey(req.Path)
+			key := l.modules.manifest.ModuleKey(req.Path)
 			record, exists := l.modules.manifest[key]
 
 			if !exists {

configs/configload/testing.go

@@ -4,9 +4,6 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
-
-	"github.com/hashicorp/terraform/configs"
-	"github.com/hashicorp/terraform/tfdiags"
 )
 
 // NewLoaderForTests is a variant of NewLoader that is intended to be more
@@ -44,58 +41,3 @@ func NewLoaderForTests(t *testing.T) (*Loader, func()) {
 
 	return loader, cleanup
 }
-
-// LoadConfigForTests is a convenience wrapper around NewLoaderForTests,
-// Loader.InstallModules and Loader.LoadConfig that allows a test configuration
-// to be loaded in a single step.
-//
-// If module installation fails, t.Fatal (or similar) is called to halt
-// execution of the test, under the assumption that installation failures are
-// not expected. If installation failures _are_ expected then use
-// NewLoaderForTests and work with the loader object directly. If module
-// installation succeeds but generates warnings, these warnings are discarded.
-//
-// If installation succeeds but errors are detected during loading then a
-// possibly-incomplete config is returned along with error diagnostics. The
-// test run is not aborted in this case, so that the caller can make assertions
-// against the returned diagnostics.
-//
-// As with NewLoaderForTests, a cleanup function is returned which must be
-// called before the test completes in order to remove the temporary
-// modules directory.
-func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func(), tfdiags.Diagnostics) {
-	t.Helper()
-
-	var diags tfdiags.Diagnostics
-
-	loader, cleanup := NewLoaderForTests(t)
-	hclDiags := loader.InstallModules(rootDir, true, InstallHooksImpl{})
-	if diags.HasErrors() {
-		cleanup()
-		diags = diags.Append(hclDiags)
-		t.Fatal(diags.Err())
-		return nil, nil, cleanup, diags
-	}
-
-	config, hclDiags := loader.LoadConfig(rootDir)
-	diags = diags.Append(hclDiags)
-	return config, loader, cleanup, diags
-}
-
-// MustLoadConfigForTests is a variant of LoadConfigForTests which calls
-// t.Fatal (or similar) if there are any errors during loading, and thus
-// does not return diagnostics at all.
-//
-// This is useful for concisely writing tests that don't expect errors at
-// all. For tests that expect errors and need to assert against them, use
-// LoadConfigForTests instead.
-func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func()) {
-	t.Helper()
-
-	config, loader, cleanup, diags := LoadConfigForTests(t, rootDir)
-	if diags.HasErrors() {
-		cleanup()
-		t.Fatal(diags.Err())
-	}
-	return config, loader, cleanup
-}

helper/resource/testing.go

@@ -26,6 +26,7 @@ import (
 	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/helper/logging"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"github.com/hashicorp/terraform/providers"
 	"github.com/hashicorp/terraform/states"
 	"github.com/hashicorp/terraform/terraform"
@@ -832,6 +833,12 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err
 		return nil, fmt.Errorf("Error creating child modules directory: %s", err)
 	}
 
+	inst := initwd.NewModuleInstaller(modulesDir, nil)
+	installDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{})
+	if installDiags.HasErrors() {
+		return nil, installDiags.Err()
+	}
+
 	loader, err := configload.NewLoader(&configload.Config{
 		ModulesDir: modulesDir,
 	})
@@ -839,11 +846,6 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err
 		return nil, fmt.Errorf("failed to create config loader: %s", err)
 	}
 
-	installDiags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{})
-	if installDiags.HasErrors() {
-		return nil, installDiags
-	}
-
 	config, configDiags := loader.LoadConfig(cfgPath)
 	if configDiags.HasErrors() {
 		return nil, configDiags

internal/earlyconfig/config_build.go

@@ -3,6 +3,7 @@ package earlyconfig
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
@@ -41,15 +42,19 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config,
 		copy(path, parent.Path)
 		path[len(path)-1] = call.Name
 
-		vc, err := version.NewConstraint(call.Version)
+		var vc version.Constraints
+		if strings.TrimSpace(call.Version) != "" {
+			var err error
+			vc, err = version.NewConstraint(call.Version)
 			if err != nil {
 				diags = diags.Append(wrapDiagnostic(tfconfig.Diagnostic{
 					Severity: tfconfig.DiagError,
 					Summary:  "Invalid version constraint",
-				Detail:   fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint: %s.", callName, call.Pos.Filename, call.Pos.Line, err),
+					Detail:   fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint %q: %s.", callName, call.Pos.Filename, call.Pos.Line, call.Version, err),
 				}))
 				continue
 			}
+		}
 
 		req := ModuleRequest{
 			Name:               call.Name,

internal/initwd/copy_dir.go

@@ -1,4 +1,4 @@
-package init
+package initwd
 
 import (
 	"io"

internal/initwd/doc.go

@@ -1,7 +1,7 @@
-// Package init contains various helper functions used by the "terraform init"
+// Package initwd contains various helper functions used by the "terraform init"
 // command.
 //
 // These functions may also be used from testing code to simulate the behaviors
 // of "terraform init" against test fixtures, but should not be used elsewhere
 // in the main code.
-package init
+package initwd

internal/initwd/from_module.go

@@ -1,7 +1,9 @@
-package configload
+package initwd
 
 import (
 	"fmt"
+	"github.com/hashicorp/terraform/internal/earlyconfig"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -9,14 +11,16 @@ import (
 	"strings"
 
 	version "github.com/hashicorp/go-version"
-	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform-config-inspect/tfconfig"
-	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/internal/modsdir"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
 const initFromModuleRootCallName = "root"
 const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "."
 
-// InitDirFromModule populates the given directory (which must exist and be
+// DirFromModule populates the given directory (which must exist and be
 // empty) with the contents of the module at the given source address.
 //
 // It does this by installing the given module and all of its descendent
@@ -34,13 +38,8 @@ const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "."
 // references using ../ from that module to be unresolvable. Error diagnostics
 // are produced in that case, to prompt the user to rewrite the source strings
 // to be absolute references to the original remote module.
-//
+func DirFromModule(rootDir, modulesDir, sourceAddr string, reg *registry.Client, hooks ModuleInstallHooks) tfdiags.Diagnostics {
-// This can be installed only on a loder that can install modules, and will
+	var diags tfdiags.Diagnostics
-// panic otherwise. Use CanInstallModules to determine if this method can be
-// used, or refer to the documentation of that method for situations where
-// install ability is guaranteed.
-func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHooks) hcl.Diagnostics {
-	var diags hcl.Diagnostics
 
 	// The way this function works is pretty ugly, but we accept it because
 	// -from-module is a less important case than normal module installation
@@ -49,20 +48,20 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 
 	// The target directory must exist but be empty.
 	{
-		entries, err := l.modules.FS.ReadDir(rootDir)
+		entries, err := ioutil.ReadDir(rootDir)
 		if err != nil {
 			if os.IsNotExist(err) {
-				diags = append(diags, &hcl.Diagnostic{
+				diags = diags.Append(tfdiags.Sourceless(
-					Severity: hcl.DiagError,
+					tfdiags.Error,
-					Summary:  "Target directory does not exist",
+					"Target directory does not exist",
-					Detail:   fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir),
+					fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir),
-				})
+				))
 			} else {
-				diags = append(diags, &hcl.Diagnostic{
+				diags = diags.Append(tfdiags.Sourceless(
-					Severity: hcl.DiagError,
+					tfdiags.Error,
-					Summary:  "Failed to read target directory",
+					"Failed to read target directory",
-					Detail:   fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err),
+					fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err),
-				})
+				))
 			}
 			return diags
 		}
@@ -74,57 +73,36 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 			haveEntries = true
 		}
 		if haveEntries {
-			diags = append(diags, &hcl.Diagnostic{
+			diags = diags.Append(tfdiags.Sourceless(
-				Severity: hcl.DiagError,
+				tfdiags.Error,
-				Summary:  "Can't populate non-empty directory",
+				"Can't populate non-empty directory",
-				Detail:   fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir),
+				fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir),
-			})
+			))
 			return diags
 		}
 	}
 
-	// We use a hidden sub-loader to manage our inner installation directory,
+	instDir := filepath.Join(rootDir, ".terraform/init-from-module")
-	// but it shares dependencies with the receiver to allow it to access the
+	inst := NewModuleInstaller(instDir, reg)
-	// same remote resources and ensure it populates the same source code
+	log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddr)
-	// cache in case .
+	os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too
-	subLoader := &Loader{
+	err := os.MkdirAll(instDir, os.ModePerm)
-		parser:  l.parser,
-		modules: l.modules, // this is a shallow copy, so we can safely mutate below
-	}
-
-	// Our sub-loader will have its own independent manifest and install
-	// directory, so we can install with it and know we won't interfere
-	// with the receiver.
-	subLoader.modules.manifest = make(moduleManifest)
-	subLoader.modules.Dir = filepath.Join(rootDir, ".terraform/init-from-module")
-
-	log.Printf("[DEBUG] using a child module loader in %s to initialize working directory from %q", subLoader.modules.Dir, sourceAddr)
-
-	subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) // if this fails then we'll fail on MkdirAll below too
-
-	err := subLoader.modules.FS.MkdirAll(subLoader.modules.Dir, os.ModePerm)
 	if err != nil {
-		diags = append(diags, &hcl.Diagnostic{
+		diags = diags.Append(tfdiags.Sourceless(
-			Severity: hcl.DiagError,
+			tfdiags.Error,
-			Summary:  "Failed to create temporary directory",
+			"Failed to create temporary directory",
-			Detail:   fmt.Sprintf("Failed to create temporary directory %s: %s.", subLoader.modules.Dir, err),
+			fmt.Sprintf("Failed to create temporary directory %s: %s.", instDir, err),
-		})
+		))
 		return diags
 	}
 
+	instManifest := make(modsdir.Manifest)
+	retManifest := make(modsdir.Manifest)
+
 	fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr)
-	fakeRange := hcl.Range{
+	fakePos := tfconfig.SourcePos{
 		Filename: fakeFilename,
-		Start: hcl.Pos{
 		Line:     1,
-			Column: 1,
-			Byte:   0,
-		},
-		End: hcl.Pos{
-			Line:   1,
-			Column: len(fakeFilename) + 1, // not accurate if the address contains unicode, but irrelevant since we have no source cache for this anyway
-			Byte:   len(fakeFilename),
-		},
 	}
 
 	// -from-module allows relative paths but it's different than a normal
@@ -146,16 +124,12 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 
 	// Now we need to create an artificial root module that will seed our
 	// installation process.
-	fakeRootModule := &configs.Module{
+	fakeRootModule := &tfconfig.Module{
-		ModuleCalls: map[string]*configs.ModuleCall{
+		ModuleCalls: map[string]*tfconfig.ModuleCall{
-			initFromModuleRootCallName: &configs.ModuleCall{
+			initFromModuleRootCallName: {
 				Name:   initFromModuleRootCallName,
-
+				Source: sourceAddr,
-				SourceAddr:      sourceAddr,
+				Pos:    fakePos,
-				SourceAddrRange: fakeRange,
-				SourceSet:       true,
-
-				DeclRange: fakeRange,
 			},
 		},
 	}
@@ -167,7 +141,7 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 		Wrapped: hooks,
 	}
 	getter := reusingGetter{}
-	instDiags := subLoader.installDescendentModules(fakeRootModule, rootDir, true, wrapHooks, getter)
+	instDiags := inst.installDescendentModules(fakeRootModule, rootDir, instManifest, true, wrapHooks, getter)
 	diags = append(diags, instDiags...)
 	if instDiags.HasErrors() {
 		return diags
@@ -175,26 +149,24 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 
 	// If all of that succeeded then we'll now migrate what was installed
 	// into the final directory structure.
-	modulesDir := l.modules.Dir
+	err = os.MkdirAll(modulesDir, os.ModePerm)
-	err = subLoader.modules.FS.MkdirAll(modulesDir, os.ModePerm)
 	if err != nil {
-		diags = append(diags, &hcl.Diagnostic{
+		diags = diags.Append(tfdiags.Sourceless(
-			Severity: hcl.DiagError,
+			tfdiags.Error,
-			Summary:  "Failed to create local modules directory",
+			"Failed to create local modules directory",
-			Detail:   fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err),
+			fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err),
-		})
+		))
 		return diags
 	}
 
-	manifest := subLoader.modules.manifest
+	recordKeys := make([]string, 0, len(instManifest))
-	recordKeys := make([]string, 0, len(manifest))
+	for k := range instManifest {
-	for k := range manifest {
 		recordKeys = append(recordKeys, k)
 	}
 	sort.Strings(recordKeys)
 
 	for _, recordKey := range recordKeys {
-		record := manifest[recordKey]
+		record := instManifest[recordKey]
 
 		if record.Key == initFromModuleRootCallName {
 			// We've found the module the user requested, which we must
@@ -202,11 +174,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 			log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir)
 			err := copyDir(rootDir, record.Dir)
 			if err != nil {
-				diags = append(diags, &hcl.Diagnostic{
+				diags = diags.Append(tfdiags.Sourceless(
-					Severity: hcl.DiagError,
+					tfdiags.Error,
-					Summary:  "Failed to copy root module",
+					"Failed to copy root module",
-					Detail:   fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err),
+					fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err),
-				})
+				))
 				continue
 			}
 
@@ -215,11 +187,12 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 			// and must thus be rewritten to be absolute addresses again.
 			// For now we can't do this rewriting automatically, but we'll
 			// generate an error to help the user do it manually.
-			mod, _ := l.parser.LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway
+			mod, _ := earlyconfig.LoadModule(rootDir) // ignore diagnostics since we're just doing value-add here anyway
+			if mod != nil {
 				for _, mc := range mod.ModuleCalls {
-				if pathTraversesUp(sourceAddr) {
+					if pathTraversesUp(mc.Source) {
 						packageAddr, givenSubdir := splitAddrSubdir(sourceAddr)
-					newSubdir := filepath.Join(givenSubdir, mc.SourceAddr)
+						newSubdir := filepath.Join(givenSubdir, mc.Source)
 						if pathTraversesUp(newSubdir) {
 							// This should never happen in any reasonable
 							// configuration since this suggests a path that
@@ -234,17 +207,17 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 						if newSubdir != "" {
 							newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir))
 						}
-					diags = append(diags, &hcl.Diagnostic{
+						diags = diags.Append(tfdiags.Sourceless(
-						Severity: hcl.DiagError,
+							tfdiags.Error,
-						Summary:  "Root module references parent directory",
+							"Root module references parent directory",
-						Detail:   fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr),
+							fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr),
-						Subject:  &mc.SourceAddrRange,
+						))
-					})
 						continue
 					}
 				}
+			}
 
-			l.modules.manifest[""] = moduleRecord{
+			retManifest[""] = modsdir.Record{
 				Key: "",
 				Dir: rootDir,
 			}
@@ -259,8 +232,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 		}
 
 		newKey := record.Key[len(initFromModuleRootKeyPrefix):]
-		instPath := filepath.Join(l.modules.Dir, newKey)
+		instPath := filepath.Join(modulesDir, newKey)
-		tempPath := filepath.Join(subLoader.modules.Dir, record.Key)
+		tempPath := filepath.Join(instDir, record.Key)
 
 		// tempPath won't be present for a module that was installed from
 		// a relative path, so in that case we just record the installation
@@ -268,11 +241,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 		// of its parent.
 		if _, err := os.Stat(tempPath); err != nil {
 			if !os.IsNotExist(err) {
-				diags = append(diags, &hcl.Diagnostic{
+				diags = diags.Append(tfdiags.Sourceless(
-					Severity: hcl.DiagError,
+					tfdiags.Error,
-					Summary:  "Failed to stat temporary module install directory",
+					"Failed to stat temporary module install directory",
-					Detail:   fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err),
+					fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err),
-				})
+				))
 				continue
 			}
 
@@ -283,8 +256,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 				parentKey = "" // parent is the root module
 			}
 
-			parentOld := manifest[initFromModuleRootKeyPrefix+parentKey]
+			parentOld := instManifest[initFromModuleRootKeyPrefix+parentKey]
-			parentNew := l.modules.manifest[parentKey]
+			parentNew := retManifest[parentKey]
 
 			// We need to figure out which portion of our directory is the
 			// parent package path and which portion is the subdirectory
@@ -301,18 +274,18 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 			newRecord := record // shallow copy
 			newRecord.Dir = newDir
 			newRecord.Key = newKey
-			l.modules.manifest[newKey] = newRecord
+			retManifest[newKey] = newRecord
 			hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
 			continue
 		}
 
-		err = subLoader.modules.FS.MkdirAll(instPath, os.ModePerm)
+		err = os.MkdirAll(instPath, os.ModePerm)
 		if err != nil {
-			diags = append(diags, &hcl.Diagnostic{
+			diags = diags.Append(tfdiags.Sourceless(
-				Severity: hcl.DiagError,
+				tfdiags.Error,
-				Summary:  "Failed to create module install directory",
+				"Failed to create module install directory",
-				Detail:   fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err),
+				fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err),
-			})
+			))
 			continue
 		}
 
@@ -321,11 +294,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 		log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath)
 		err := copyDir(instPath, tempPath)
 		if err != nil {
-			diags = append(diags, &hcl.Diagnostic{
+			diags = diags.Append(tfdiags.Sourceless(
-				Severity: hcl.DiagError,
+				tfdiags.Error,
-				Summary:  "Failed to copy descendent module",
+				"Failed to copy descendent module",
-				Detail:   fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err),
+				fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err),
-			})
+			))
 			continue
 		}
 
@@ -339,23 +312,23 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook
 		newRecord := record // shallow copy
 		newRecord.Dir = filepath.Join(instPath, subDir)
 		newRecord.Key = newKey
-		l.modules.manifest[newKey] = newRecord
+		retManifest[newKey] = newRecord
 		hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir)
 	}
 
-	err = l.modules.writeModuleManifestSnapshot()
+	retManifest.WriteSnapshotToDir(modulesDir)
 	if err != nil {
-		diags = append(diags, &hcl.Diagnostic{
+		diags = diags.Append(tfdiags.Sourceless(
-			Severity: hcl.DiagError,
+			tfdiags.Error,
-			Summary:  "Failed to write module manifest",
+			"Failed to write module manifest",
-			Detail:   fmt.Sprintf("Error writing module manifest: %s.", err),
+			fmt.Sprintf("Error writing module manifest: %s.", err),
-		})
+		))
 	}
 
 	if !diags.HasErrors() {
 		// Try to clean up our temporary directory, but don't worry if we don't
 		// succeed since it shouldn't hurt anything.
-		subLoader.modules.FS.RemoveAll(subLoader.modules.Dir)
+		os.RemoveAll(instDir)
 	}
 
 	return diags
@@ -373,8 +346,8 @@ func pathTraversesUp(path string) bool {
 // does its own installation steps after the initial installation pass
 // has completed.
 type installHooksInitDir struct {
-	Wrapped InstallHooks
+	Wrapped ModuleInstallHooks
-	InstallHooksImpl
+	ModuleInstallHooksImpl
 }
 
 func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) {

internal/initwd/from_module_test.go

@@ -1,4 +1,4 @@
-package configload
+package initwd
 
 import (
 	"os"
@@ -8,20 +8,25 @@ import (
 
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
-func TestLoaderInitDirFromModule_registry(t *testing.T) {
+func TestDirFromModule_registry(t *testing.T) {
 	if os.Getenv("TF_ACC") == "" {
 		t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
 	}
 
 	fixtureDir := filepath.Clean("test-fixtures/empty")
-	loader, done := tempChdirLoader(t, fixtureDir)
+	dir, done := tempChdir(t, fixtureDir)
+	modsDir := filepath.Join(dir, ".terraform/modules")
 	defer done()
 
 	hooks := &testInstallHooks{}
 
-	diags := loader.InitDirFromModule(".", "hashicorp/module-installer-acctest/aws//examples/main", hooks)
+	reg := registry.NewClient(nil, nil)
+	diags := DirFromModule(dir, modsDir, "hashicorp/module-installer-acctest/aws//examples/main", reg, hooks)
 	assertNoDiagnostics(t, diags)
 
 	v := version.Must(version.NewVersion("0.0.1"))
@@ -65,10 +70,17 @@ func TestLoaderInitDirFromModule_registry(t *testing.T) {
 		return
 	}
 
+	loader, err := configload.NewLoader(&configload.Config{
+		ModulesDir: modsDir,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Make sure the configuration is loadable now.
 	// (This ensures that correct information is recorded in the manifest.)
 	config, loadDiags := loader.LoadConfig(".")
-	if assertNoDiagnostics(t, loadDiags) {
+	if assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) {
 		return
 	}
 

internal/initwd/getter.go

@@ -1,4 +1,4 @@
-package init
+package initwd
 
 import (
 	"fmt"
@@ -23,6 +23,7 @@ var goGetterDetectors = []getter.Detector{
 	new(getter.GitHubDetector),
 	new(getter.BitBucketDetector),
 	new(getter.S3Detector),
+	new(getter.FileDetector),
 }
 
 var goGetterNoDetectors = []getter.Detector{}
@@ -44,6 +45,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{
 }
 
 var goGetterGetters = map[string]getter.Getter{
+	"file":  new(getter.FileGetter),
 	"git":   new(getter.GitGetter),
 	"hg":    new(getter.HgGetter),
 	"s3":    new(getter.S3Getter),

internal/initwd/inode.go

@@ -1,6 +1,6 @@
 // +build linux darwin openbsd netbsd solaris dragonfly
 
-package init
+package initwd
 
 import (
 	"fmt"

internal/initwd/inode_freebsd.go

@@ -1,6 +1,6 @@
 // +build freebsd
 
-package init
+package initwd
 
 import (
 	"fmt"

internal/initwd/inode_windows.go

@@ -1,6 +1,6 @@
 // +build windows
 
-package init
+package initwd
 
 // no syscall.Stat_t on windows, return 0 for inodes
 func inode(path string) (uint64, error) {

internal/initwd/module_install.go

@@ -1,4 +1,4 @@
-package init
+package initwd
 
 import (
 	"fmt"
@@ -58,7 +58,9 @@ func NewModuleInstaller(modsDir string, reg *registry.Client) *ModuleInstaller {
 // Use CanInstallModules to determine if a loader can install modules, or
 // refer to the documentation for that method for situations where module
 // installation capability is guaranteed.
-func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) tfdiags.Diagnostics {
+func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks ModuleInstallHooks) tfdiags.Diagnostics {
+	log.Printf("[TRACE] ModuleInstaller: installing child modules for %s into %s", rootDir, i.modsDir)
+
 	rootMod, diags := earlyconfig.LoadModule(rootDir)
 	if rootMod == nil {
 		return diags
@@ -81,12 +83,12 @@ func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks Ins
 	return diags
 }
 
-func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks InstallHooks, getter reusingGetter) tfdiags.Diagnostics {
+func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks ModuleInstallHooks, getter reusingGetter) tfdiags.Diagnostics {
 	var diags tfdiags.Diagnostics
 
 	if hooks == nil {
 		// Use our no-op implementation as a placeholder
-		hooks = InstallHooksImpl{}
+		hooks = ModuleInstallHooksImpl{}
 	}
 
 	// Create a manifest record for the root module. This will be used if
@@ -111,13 +113,13 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 				record, recorded := manifest[key]
 				switch {
 				case !recorded:
-					log.Printf("[TRACE] %s is not yet installed", key)
+					log.Printf("[TRACE] ModuleInstaller: %s is not yet installed", key)
 					replace = true
 				case record.SourceAddr != req.SourceAddr:
-					log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
+					log.Printf("[TRACE] ModuleInstaller: %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
 					replace = true
 				case record.Version != nil && !req.VersionConstraints.Check(record.Version):
-					log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints)
+					log.Printf("[TRACE] ModuleInstaller: %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints)
 					replace = true
 				}
 			}
@@ -127,7 +129,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 			// existing remnants.
 			if replace {
 				if _, recorded := manifest[key]; recorded {
-					log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
+					log.Printf("[TRACE] ModuleInstaller: discarding previous record of %s prior to reinstall", key)
 				}
 				delete(manifest, key)
 				// Deleting a module invalidates all of its descendent modules too.
@@ -135,7 +137,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 				for subKey := range manifest {
 					if strings.HasPrefix(subKey, keyPrefix) {
 						if _, recorded := manifest[subKey]; recorded {
-							log.Printf("[TRACE] also discarding downstream %s", subKey)
+							log.Printf("[TRACE] ModuleInstaller: also discarding downstream %s", subKey)
 						}
 						delete(manifest, subKey)
 					}
@@ -147,10 +149,10 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 				// Clean up any stale cache directory that might be present.
 				// If this is a local (relative) source then the dir will
 				// not exist, but we'll ignore that.
-				log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
+				log.Printf("[TRACE] ModuleInstaller: cleaning directory %s prior to install of %s", instPath, key)
 				err := os.RemoveAll(instPath)
 				if err != nil && !os.IsNotExist(err) {
-					log.Printf("[TRACE] failed to remove %s: %s", key, err)
+					log.Printf("[TRACE] ModuleInstaller: failed to remove %s: %s", key, err)
 					diags = diags.Append(tfdiags.Sourceless(
 						tfdiags.Error,
 						"Failed to remove local module cache",
@@ -170,7 +172,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 					mod, mDiags := earlyconfig.LoadModule(record.Dir)
 					diags = diags.Append(mDiags)
 
-					log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
+					log.Printf("[TRACE] ModuleInstaller: Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
 					return mod, record.Version, diags
 				}
 			}
@@ -181,7 +183,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 			switch {
 
 			case isLocalSourceAddr(req.SourceAddr):
-				log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
+				log.Printf("[TRACE] ModuleInstaller: %s has local path %q", key, req.SourceAddr)
 				mod, mDiags := i.installLocalModule(req, key, manifest, hooks)
 				diags = append(diags, mDiags...)
 				return mod, nil, diags
@@ -192,14 +194,14 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 					// Should never happen because isRegistrySourceAddr already validated
 					panic(err)
 				}
-				log.Printf("[TRACE] %s is a registry module at %s", key, addr)
+				log.Printf("[TRACE] ModuleInstaller: %s is a registry module at %s", key, addr)
 
 				mod, v, mDiags := i.installRegistryModule(req, key, instPath, addr, manifest, hooks, getter)
 				diags = append(diags, mDiags...)
 				return mod, v, diags
 
 			default:
-				log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
+				log.Printf("[TRACE] ModuleInstaller: %s address %q will be handled by go-getter", key, req.SourceAddr)
 
 				mod, mDiags := i.installGoGetterModule(req, key, instPath, manifest, hooks, getter)
 				diags = append(diags, mDiags...)
@@ -222,7 +224,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo
 	return diags
 }
 
-func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks InstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) {
+func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks ModuleInstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	parentKey := manifest.ModuleKey(req.Parent.Path)
@@ -244,7 +246,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key
 	// filesystem at all because the parent already wrote
 	// the files we need, and so we just load up what's already here.
 	newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
-	log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
+	log.Printf("[TRACE] ModuleInstaller: %s uses directory from parent: %s", key, newDir)
 	mod, mDiags := earlyconfig.LoadModule(newDir)
 	if mod == nil {
 		// nil indicates missing or unreadable directory, so we'll
@@ -271,7 +273,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key
 	return mod, diags
 }
 
-func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
+func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	hostname, err := addr.SvcHost()
@@ -353,7 +355,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 		// If we've found a pre-release version then we'll ignore it unless
 		// it was exactly requested.
 		if v.Prerelease() != "" && req.VersionConstraints.String() != v.String() {
-			log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
+			log.Printf("[TRACE] ModuleInstaller: %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
 			continue
 		}
 
@@ -404,7 +406,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 		return nil, nil, diags
 	}
 
-	log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
+	log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
 
 	modDir, err := getter.getWithGoGetter(instPath, dlAddr)
 	if err != nil {
@@ -421,7 +423,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 		return nil, nil, diags
 	}
 
-	log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
+	log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, dlAddr, modDir)
 
 	if addr.RawSubmodule != "" {
 		// Append the user's requested subdirectory to any subdirectory that
@@ -429,7 +431,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 		modDir = filepath.Join(modDir, addr.RawSubmodule)
 	}
 
-	log.Printf("[TRACE] %s should now be at %s", key, modDir)
+	log.Printf("[TRACE] ModuleInstaller: %s should now be at %s", key, modDir)
 
 	// Finally we are ready to try actually loading the module.
 	mod, mDiags := earlyconfig.LoadModule(modDir)
@@ -461,7 +463,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest,
 	return mod, latestMatch, diags
 }
 
-func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) {
+func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 
 	// Report up to the caller that we're about to start downloading.
@@ -483,7 +485,7 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest,
 		return nil, diags
 	}
 
-	log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
+	log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, req.SourceAddr, modDir)
 
 	mod, mDiags := earlyconfig.LoadModule(modDir)
 	if mod == nil {

internal/initwd/module_install_hooks.go

@@ -1,16 +1,16 @@
-package init
+package initwd
 
 import (
 	version "github.com/hashicorp/go-version"
 )
 
-// InstallHooks is an interface used to provide notifications about the
+// ModuleInstallHooks is an interface used to provide notifications about the
 // installation process being orchestrated by InstallModules.
 //
 // This interface may have new methods added in future, so implementers should
 // embed InstallHooksImpl to get no-op implementations of any unimplemented
 // methods.
-type InstallHooks interface {
+type ModuleInstallHooks interface {
 	// Download is called for modules that are retrieved from a remote source
 	// before that download begins, to allow a caller to give feedback
 	// on progress through a possibly-long sequence of downloads.
@@ -21,16 +21,16 @@ type InstallHooks interface {
 	Install(moduleAddr string, version *version.Version, localPath string)
 }
 
-// InstallHooksImpl is a do-nothing implementation of InstallHooks that
+// ModuleInstallHooksImpl is a do-nothing implementation of InstallHooks that
 // can be embedded in another implementation struct to allow only partial
 // implementation of the interface.
-type InstallHooksImpl struct {
+type ModuleInstallHooksImpl struct {
 }
 
-func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
+func (h ModuleInstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
 }
 
-func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
+func (h ModuleInstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
 }
 
-var _ InstallHooks = InstallHooksImpl{}
+var _ ModuleInstallHooks = ModuleInstallHooksImpl{}

internal/initwd/module_install_test.go

@@ -1,23 +1,47 @@
-package configload
+package initwd
 
 import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/go-test/deep"
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/helper/logging"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/tfdiags"
 )
 
-func TestLoaderInstallModules_local(t *testing.T) {
+func TestMain(m *testing.M) {
+	flag.Parse()
+	if testing.Verbose() {
+		// if we're verbose, use the logging requested by TF_LOG
+		logging.SetOutput()
+	} else {
+		// otherwise silence all logs
+		log.SetOutput(ioutil.Discard)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestModuleInstaller(t *testing.T) {
 	fixtureDir := filepath.Clean("test-fixtures/local-modules")
-	loader, done := tempChdirLoader(t, fixtureDir)
+	dir, done := tempChdir(t, fixtureDir)
 	defer done()
 
 	hooks := &testInstallHooks{}
 
-	diags := loader.InstallModules(".", false, hooks)
+	modulesDir := filepath.Join(dir, ".terraform/modules")
+	inst := NewModuleInstaller(modulesDir, nil)
+	diags := inst.InstallModules(".", false, hooks)
 	assertNoDiagnostics(t, diags)
 
 	wantCalls := []testInstallHookCall{
@@ -39,10 +63,17 @@ func TestLoaderInstallModules_local(t *testing.T) {
 		return
 	}
 
+	loader, err := configload.NewLoader(&configload.Config{
+		ModulesDir: modulesDir,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Make sure the configuration is loadable now.
 	// (This ensures that correct information is recorded in the manifest.)
 	config, loadDiags := loader.LoadConfig(".")
-	assertNoDiagnostics(t, loadDiags)
+	assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
 
 	wantTraces := map[string]string{
 		"":                "in root module",
@@ -67,13 +98,14 @@ func TestLoaderInstallModules_registry(t *testing.T) {
 		t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it")
 	}
 
-	fixtureDir := filepath.Clean("test-fixtures/registry-modules")
+	fixtureDir := filepath.Clean("test-fixtures/local-modules")
-	loader, done := tempChdirLoader(t, fixtureDir)
+	dir, done := tempChdir(t, fixtureDir)
 	defer done()
 
 	hooks := &testInstallHooks{}
-
+	modulesDir := filepath.Join(dir, ".terraform/modules")
-	diags := loader.InstallModules(".", false, hooks)
+	inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil))
+	diags := inst.InstallModules(dir, false, hooks)
 	assertNoDiagnostics(t, diags)
 
 	v := version.Must(version.NewVersion("0.0.1"))
@@ -153,10 +185,17 @@ func TestLoaderInstallModules_registry(t *testing.T) {
 		return
 	}
 
+	loader, err := configload.NewLoader(&configload.Config{
+		ModulesDir: modulesDir,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Make sure the configuration is loadable now.
 	// (This ensures that correct information is recorded in the manifest.)
 	config, loadDiags := loader.LoadConfig(".")
-	assertNoDiagnostics(t, loadDiags)
+	assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
 
 	wantTraces := map[string]string{
 		"":                             "in local caller for registry-modules",
@@ -187,12 +226,13 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
 	}
 
 	fixtureDir := filepath.Clean("test-fixtures/go-getter-modules")
-	loader, done := tempChdirLoader(t, fixtureDir)
+	dir, done := tempChdir(t, fixtureDir)
 	defer done()
 
 	hooks := &testInstallHooks{}
-
+	modulesDir := filepath.Join(dir, ".terraform/modules")
-	diags := loader.InstallModules(".", false, hooks)
+	inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil))
+	diags := inst.InstallModules(dir, false, hooks)
 	assertNoDiagnostics(t, diags)
 
 	wantCalls := []testInstallHookCall{
@@ -264,10 +304,17 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
 		return
 	}
 
+	loader, err := configload.NewLoader(&configload.Config{
+		ModulesDir: modulesDir,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Make sure the configuration is loadable now.
 	// (This ensures that correct information is recorded in the manifest.)
 	config, loadDiags := loader.LoadConfig(".")
-	assertNoDiagnostics(t, loadDiags)
+	assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
 
 	wantTraces := map[string]string{
 		"":                             "in local caller for go-getter-modules",
@@ -321,3 +368,97 @@ func (h *testInstallHooks) Install(moduleAddr string, version *version.Version,
 		LocalPath:  localPath,
 	})
 }
+
+// tempChdir copies the contents of the given directory to a temporary
+// directory and changes the test process's current working directory to
+// point to that directory. Also returned is a function that should be
+// called at the end of the test (e.g. via "defer") to restore the previous
+// working directory.
+//
+// Tests using this helper cannot safely be run in parallel with other tests.
+func tempChdir(t *testing.T, sourceDir string) (string, func()) {
+	t.Helper()
+
+	tmpDir, err := ioutil.TempDir("", "terraform-configload")
+	if err != nil {
+		t.Fatalf("failed to create temporary directory: %s", err)
+		return "", nil
+	}
+
+	if err := copyDir(tmpDir, sourceDir); err != nil {
+		t.Fatalf("failed to copy fixture to temporary directory: %s", err)
+		return "", nil
+	}
+
+	oldDir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("failed to determine current working directory: %s", err)
+		return "", nil
+	}
+
+	err = os.Chdir(tmpDir)
+	if err != nil {
+		t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err)
+		return "", nil
+	}
+
+	// Most of the tests need this, so we'll make it just in case.
+	os.MkdirAll(filepath.Join(tmpDir, ".terraform/modules"), os.ModePerm)
+
+	t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir)
+
+	return tmpDir, func() {
+		err := os.Chdir(oldDir)
+		if err != nil {
+			panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err))
+		}
+
+		if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" {
+			os.RemoveAll(tmpDir)
+		}
+	}
+}
+
+func assertNoDiagnostics(t *testing.T, diags tfdiags.Diagnostics) bool {
+	t.Helper()
+	return assertDiagnosticCount(t, diags, 0)
+}
+
+func assertDiagnosticCount(t *testing.T, diags tfdiags.Diagnostics, want int) bool {
+	t.Helper()
+	if len(diags) != 0 {
+		t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want)
+		for _, diag := range diags {
+			t.Logf("- %s", diag)
+		}
+		return true
+	}
+	return false
+}
+
+func assertDiagnosticSummary(t *testing.T, diags tfdiags.Diagnostics, want string) bool {
+	t.Helper()
+
+	for _, diag := range diags {
+		if diag.Description().Summary == want {
+			return false
+		}
+	}
+
+	t.Errorf("missing diagnostic summary %q", want)
+	for _, diag := range diags {
+		t.Logf("- %s", diag)
+	}
+	return true
+}
+
+func assertResultDeepEqual(t *testing.T, got, want interface{}) bool {
+	t.Helper()
+	if diff := deep.Equal(got, want); diff != nil {
+		for _, problem := range diff {
+			t.Errorf("%s", problem)
+		}
+		return true
+	}
+	return false
+}

internal/initwd/test-fixtures/already-installed/root.tf

@@ -0,0 +1,10 @@
+
+module "child_a" {
+  source  = "example.com/foo/bar_a/baz"
+  version = ">= 1.0.0"
+}
+
+module "child_b" {
+  source = "example.com/foo/bar_b/baz"
+  version = ">= 1.0.0"
+}

internal/initwd/test-fixtures/go-getter-modules/.gitignore

@@ -0,0 +1 @@
+.terraform/*

internal/initwd/test-fixtures/go-getter-modules/root.tf

@@ -0,0 +1,21 @@
+# This fixture depends on a github repo at:
+#     https://github.com/hashicorp/terraform-aws-module-installer-acctest
+# ...and expects its v0.0.1 tag to be pointing at the following commit:
+#     d676ab2559d4e0621d59e3c3c4cbb33958ac4608
+
+variable "v" {
+  description = "in local caller for go-getter-modules"
+  default     = ""
+}
+
+module "acctest_root" {
+  source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1"
+}
+
+module "acctest_child_a" {
+  source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1"
+}
+
+module "acctest_child_b" {
+  source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1"
+}

internal/initwd/test-fixtures/local-modules/child_a/child_a.tf

@@ -0,0 +1,9 @@
+
+variable "v" {
+  description = "in child_a module"
+  default     = ""
+}
+
+module "child_b" {
+  source = "./child_b"
+}

internal/initwd/test-fixtures/local-modules/child_a/child_b/child_b.tf

@@ -0,0 +1,9 @@
+
+variable "v" {
+  description = "in child_b module"
+  default     = ""
+}
+
+output "hello" {
+  value = "Hello from child_b!"
+}

internal/initwd/test-fixtures/local-modules/root.tf

@@ -0,0 +1,9 @@
+
+variable "v" {
+  description = "in root module"
+  default     = ""
+}
+
+module "child_a" {
+  source = "./child_a"
+}

internal/initwd/test-fixtures/registry-modules/.gitignore

@@ -0,0 +1 @@
+.terraform/*

internal/initwd/test-fixtures/registry-modules/root.tf

@@ -0,0 +1,33 @@
+# This fixture indirectly depends on a github repo at:
+#     https://github.com/hashicorp/terraform-aws-module-installer-acctest
+# ...and expects its v0.0.1 tag to be pointing at the following commit:
+#     d676ab2559d4e0621d59e3c3c4cbb33958ac4608
+#
+# This repository is accessed indirectly via:
+#     https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1
+#
+# Since the tag's id is included in a downloaded archive, it is expected to
+# have the following id:
+#     853d03855b3290a3ca491d4c3a7684572dd42237
+# (this particular assumption is encoded in the tests that use this fixture)
+
+
+variable "v" {
+  description = "in local caller for registry-modules"
+  default     = ""
+}
+
+module "acctest_root" {
+  source  = "hashicorp/module-installer-acctest/aws"
+  version = "0.0.1"
+}
+
+module "acctest_child_a" {
+  source  = "hashicorp/module-installer-acctest/aws//modules/child_a"
+  version = "0.0.1"
+}
+
+module "acctest_child_b" {
+  source  = "hashicorp/module-installer-acctest/aws//modules/child_b"
+  version = "0.0.1"
+}

internal/initwd/testing.go

@@ -0,0 +1,67 @@
+package initwd
+
+import (
+	"github.com/hashicorp/terraform/registry"
+	"testing"
+
+	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/configs/configload"
+	"github.com/hashicorp/terraform/tfdiags"
+)
+
+// LoadConfigForTests is a convenience wrapper around configload.NewLoaderForTests,
+// ModuleInstaller.InstallModules and configload.Loader.LoadConfig that allows
+// a test configuration to be loaded in a single step.
+//
+// If module installation fails, t.Fatal (or similar) is called to halt
+// execution of the test, under the assumption that installation failures are
+// not expected. If installation failures _are_ expected then use
+// NewLoaderForTests and work with the loader object directly. If module
+// installation succeeds but generates warnings, these warnings are discarded.
+//
+// If installation succeeds but errors are detected during loading then a
+// possibly-incomplete config is returned along with error diagnostics. The
+// test run is not aborted in this case, so that the caller can make assertions
+// against the returned diagnostics.
+//
+// As with NewLoaderForTests, a cleanup function is returned which must be
+// called before the test completes in order to remove the temporary
+// modules directory.
+func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func(), tfdiags.Diagnostics) {
+	t.Helper()
+
+	var diags tfdiags.Diagnostics
+
+	loader, cleanup := configload.NewLoaderForTests(t)
+	inst := NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
+
+	moreDiags := inst.InstallModules(rootDir, true, ModuleInstallHooksImpl{})
+	diags = diags.Append(moreDiags)
+	if diags.HasErrors() {
+		cleanup()
+		t.Fatal(diags.Err())
+		return nil, nil, func() {}, diags
+	}
+
+	config, hclDiags := loader.LoadConfig(rootDir)
+	diags = diags.Append(hclDiags)
+	return config, loader, cleanup, diags
+}
+
+// MustLoadConfigForTests is a variant of LoadConfigForTests which calls
+// t.Fatal (or similar) if there are any errors during loading, and thus
+// does not return diagnostics at all.
+//
+// This is useful for concisely writing tests that don't expect errors at
+// all. For tests that expect errors and need to assert against them, use
+// LoadConfigForTests instead.
+func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func()) {
+	t.Helper()
+
+	config, loader, cleanup, diags := LoadConfigForTests(t, rootDir)
+	if diags.HasErrors() {
+		cleanup()
+		t.Fatal(diags.Err())
+	}
+	return config, loader, cleanup
+}

internal/modsdir/manifest.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 
@@ -95,6 +96,9 @@ func ReadManifestSnapshotForDir(dir string) (Manifest, error) {
 	fn := filepath.Join(dir, ManifestSnapshotFilename)
 	r, err := os.Open(fn)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return make(Manifest), nil // missing file is okay and treated as empty
+		}
 		return nil, err
 	}
 	return ReadManifestSnapshot(r)
@@ -125,6 +129,7 @@ func (m Manifest) WriteSnapshot(w io.Writer) error {
 
 func (m Manifest) WriteSnapshotToDir(dir string) error {
 	fn := filepath.Join(dir, ManifestSnapshotFilename)
+	log.Printf("[TRACE] modsdir: writing modules manifest to %s", fn)
 	w, err := os.Create(fn)
 	if err != nil {
 		return err

repl/session_test.go

@@ -2,6 +2,7 @@ package repl
 
 import (
 	"flag"
+	"github.com/hashicorp/terraform/internal/initwd"
 	"io/ioutil"
 	"log"
 	"os"
@@ -11,7 +12,6 @@ import (
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/hashicorp/terraform/addrs"
-	"github.com/hashicorp/terraform/configs/configload"
 	"github.com/hashicorp/terraform/configs/configschema"
 	"github.com/hashicorp/terraform/helper/logging"
 	"github.com/hashicorp/terraform/providers"
@@ -202,17 +202,10 @@ func testSession(t *testing.T, test testSessionTest) {
 		},
 	}
 
-	loader, cleanup := configload.NewLoaderForTests(t)
+	config, _, cleanup, configDiags := initwd.LoadConfigForTests(t, "testdata/config-fixture")
 	defer cleanup()
-
-	configDiags := loader.InstallModules("testdata/config-fixture", false, nil)
 	if configDiags.HasErrors() {
-		t.Fatalf("unexpected problems initializing test config: %s", configDiags.Error())
+		t.Fatalf("unexpected problems loading config: %s", configDiags.Err())
-	}
-
-	config, configDiags := loader.LoadConfig("testdata/config-fixture")
-	if configDiags.HasErrors() {
-		t.Fatalf("unexpected problems loading config: %s", configDiags.Error())
 	}
 
 	// Build the TF context

terraform/terraform_test.go

@@ -2,6 +2,8 @@ package terraform
 
 import (
 	"flag"
+	"github.com/hashicorp/terraform/internal/initwd"
+	"github.com/hashicorp/terraform/registry"
 	"io"
 	"io/ioutil"
 	"log"
@@ -102,7 +104,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
 	t.Helper()
 
 	dir := filepath.Join(fixtureDir, name)
-
 	// FIXME: We're not dealing with the cleanup function here because
 	// this testModule function is used all over and so we don't want to
 	// change its interface at this late stage.
@@ -111,9 +112,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config
 	// Test modules usually do not refer to remote sources, and for local
 	// sources only this ultimately just records all of the module paths
 	// in a JSON file so that we can load them below.
-	diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{})
+	inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
-	if diags.HasErrors() {
+	instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{})
-		t.Fatal(diags.Error())
+	if instDiags.HasErrors() {
+		t.Fatal(instDiags.Err())
 	}
 
 	config, snap, diags := loader.LoadConfigWithSnapshot(dir)
@@ -162,9 +164,10 @@ func testModuleInline(t *testing.T, sources map[string]string) *configs.Config {
 	// Test modules usually do not refer to remote sources, and for local
 	// sources only this ultimately just records all of the module paths
 	// in a JSON file so that we can load them below.
-	diags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{})
+	inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil))
-	if diags.HasErrors() {
+	instDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{})
-		t.Fatal(diags.Error())
+	if instDiags.HasErrors() {
+		t.Fatal(instDiags.Err())
 	}
 
 	config, diags := loader.LoadConfig(cfgPath)