configs/configload: installation of local modules

Enough of the InstallModules method to install local modules (those with
relative paths). "Install" is actually a bit of an exaggeration for these
since we actually just record them in our manifest after verifying that
the source directory exists.

This is a change of behavior relative to the old module installer since
we no longer create a symlink to the module directory inside the
.terraform/modules directory. Instead, we record the module's true
location in our manifest so that the loader will find it later.

The use of a symlink here predated the manifest file. Now that we have a
manifest file the symlinks are redundant. Using the "natural" location of
the module leads to more helpful error messages, since we'll refer to
the module path as the user expects it, rather than to an internal alias.

configs/configload/loader_install.go

@@ -0,0 +1,237 @@
+package configload
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/registry/regsrc"
+)
+
+// InstallModules analyses the root module in the given directory and installs
+// all of its direct and transitive dependencies into the loader's modules
+// directory, which must already exist.
+//
+// Since InstallModules makes possibly-time-consuming calls to remote services,
+// a hook interface is supported to allow the caller to be notified when
+// each module is installed and, for remote modules, when downloading begins.
+// LoadConfig guarantees that two hook calls will not happen concurrently but
+// it does not guarantee any particular ordering of hook calls. This mechanism
+// is for UI feedback only and does not give the caller any control over the
+// process.
+//
+// If modules are already installed in the target directory, they will be
+// skipped unless their source address or version have changed or unless
+// the upgrade flag is set.
+//
+// InstallModules never deletes any directory, except in the case where it
+// needs to replace a directory that is already present with a newly-extracted
+// package.
+//
+// If the returned diagnostics contains errors then the module installation
+// may have wholly or partially completed. Modules must be loaded in order
+// to find their dependencies, so this function does many of the same checks
+// as LoadConfig as a side-effect.
+func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
+	rootMod, diags := l.parser.LoadConfigDir(rootDir)
+	if rootMod == nil {
+		return diags
+	}
+
+	if hooks == nil {
+		// Use our no-op implementation as a placeholder
+		hooks = InstallHooksImpl{}
+	}
+
+	// Create a manifest record for the root module. This will be used if
+	// there are any relative-pathed modules in the root.
+	l.modules.manifest[""] = moduleRecord{
+		Key: "",
+		Dir: rootDir,
+	}
+
+	_, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
+		func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
+
+			key := manifestKey(req.Path)
+			instPath := l.packageInstallPath(req.Path)
+
+			log.Printf("[DEBUG] Module installer: begin %s", key)
+
+			// First we'll check if we need to upgrade/replace an existing
+			// installed module, and delete it out of the way if so.
+			replace := upgrade
+			if !replace {
+				record, recorded := l.modules.manifest[key]
+				switch {
+				case !recorded:
+					log.Printf("[TRACE] %s is not yet installed", key)
+					replace = true
+				case record.SourceAddr != req.SourceAddr:
+					log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
+					replace = true
+				case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
+					log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
+					replace = true
+				}
+			}
+
+			// If we _are_ planning to replace this module, then we'll remove
+			// it now so our installation code below won't conflict with any
+			// existing remnants.
+			if replace {
+				if _, recorded := l.modules.manifest[key]; recorded {
+					log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
+				}
+				delete(l.modules.manifest, key)
+				// Deleting a module invalidates all of its descendent modules too.
+				keyPrefix := key + "."
+				for subKey := range l.modules.manifest {
+					if strings.HasPrefix(subKey, keyPrefix) {
+						if _, recorded := l.modules.manifest[subKey]; recorded {
+							log.Printf("[TRACE] also discarding downstream %s", subKey)
+						}
+						delete(l.modules.manifest, subKey)
+					}
+				}
+			}
+
+			record, recorded := l.modules.manifest[key]
+			if !recorded {
+				// Clean up any stale cache directory that might be present.
+				// If this is a local (relative) source then the dir will
+				// not exist, but we'll ignore that.
+				log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
+				err := l.modules.FS.RemoveAll(instPath)
+				if err != nil && !os.IsNotExist(err) {
+					log.Printf("[TRACE] failed to remove %s: %s", key, err)
+					diags = append(diags, &hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Failed to remove local module cache",
+						Detail: fmt.Sprintf(
+							"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
+							instPath, err,
+						),
+						Subject: &req.CallRange,
+					})
+					return nil, nil, diags
+				}
+			} else {
+				// If this module is already recorded and its root directory
+				// exists then we will just load what's already there and
+				// keep our existing record.
+				info, err := l.modules.FS.Stat(record.Dir)
+				if err == nil && info.IsDir() {
+					mod, mDiags := l.parser.LoadConfigDir(record.Dir)
+					diags = append(diags, mDiags...)
+
+					log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
+					return mod, record.Version, diags
+				}
+			}
+
+			// If we get down here then it's finally time to actually install
+			// the module. There are some variants to this process depending
+			// on what type of module source address we have.
+			switch {
+
+			case isLocalSourceAddr(req.SourceAddr):
+				log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
+				mod, mDiags := l.installLocalModule(req, key, hooks)
+				diags = append(diags, mDiags...)
+				return mod, nil, diags
+
+			case isRegistrySourceAddr(req.SourceAddr):
+				addr, err := regsrc.ParseModuleSource(req.SourceAddr)
+				if err != nil {
+					// Should never happen because isRegistrySourceAddr already validated
+					panic(err)
+				}
+				log.Printf("[TRACE] %s is a registry module at %s", key, addr)
+
+				// TODO: Implement
+				panic("registry source installation not yet implemented")
+
+			default:
+				log.Printf("[TRACE] %s address %q will be interpreted with go-getter", key, req.SourceAddr)
+
+				// TODO: Implement
+				panic("fallback source installation not yet implemented")
+
+			}
+
+		},
+	))
+	diags = append(diags, cDiags...)
+
+	err := l.modules.writeModuleManifestSnapshot()
+	if err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to update module manifest",
+			Detail:   fmt.Sprintf("Unable to write the module manifest file: %s", err),
+		})
+	}
+
+	return diags
+}
+
+func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
+	var diags hcl.Diagnostics
+
+	parentKey := manifestKey(req.Parent.Path)
+	parentRecord, recorded := l.modules.manifest[parentKey]
+	if !recorded {
+		// This is indicative of a bug rather than a user-actionable error
+		panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
+	}
+
+	if len(req.VersionConstraint.Required) != 0 {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid version constraint",
+			Detail:   "A version constraint cannot be applied to a module at a relative local path.",
+			Subject:  &req.VersionConstraint.DeclRange,
+		})
+	}
+
+	// For local sources we don't actually need to modify the
+	// filesystem at all because the parent already wrote
+	// the files we need, and so we just load up what's already here.
+	newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
+	log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
+	mod, mDiags := l.parser.LoadConfigDir(newDir)
+	if mod == nil {
+		// nil indicates missing or unreadable directory, so we'll
+		// discard the returned diags and return a more specific
+		// error message here.
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Unreadable module directory",
+			Detail:   fmt.Sprintf("The directory %s could not be read.", newDir),
+			Subject:  &req.SourceAddrRange,
+		})
+	} else {
+		diags = append(diags, mDiags...)
+	}
+
+	// Note the local location in our manifest.
+	l.modules.manifest[key] = moduleRecord{
+		Key:        key,
+		Dir:        newDir,
+		SourceAddr: req.SourceAddr,
+	}
+	log.Printf("[TRACE] Module installer: %s installed at %s", key, newDir)
+	hooks.Install(key, nil, newDir)
+
+	return mod, diags
+}
+
+func (l *Loader) packageInstallPath(modulePath []string) string {
+	return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
+}

configs/configload/loader_install_hooks.go

@@ -0,0 +1,34 @@
+package configload
+
+import version "github.com/hashicorp/go-version"
+
+// InstallHooks is an interface used to provide notifications about the
+// installation process being orchestrated by InstallModules.
+//
+// This interface may have new methods added in future, so implementers should
+// embed InstallHooksImpl to get no-op implementations of any unimplemented
+// methods.
+type InstallHooks interface {
+	// Download is called for modules that are retrieved from a remote source
+	// before that download begins, to allow a caller to give feedback
+	// on progress through a possibly-long sequence of downloads.
+	Download(moduleAddr, packageAddr string, version *version.Version)
+
+	// Install is called for each module that is installed, even if it did
+	// not need to be downloaded from a remote source.
+	Install(moduleAddr string, version *version.Version, localPath string)
+}
+
+// InstallHooksImpl is a do-nothing implementation of InstallHooks that
+// can be embedded in another implementation struct to allow only partial
+// implementation of the interface.
+type InstallHooksImpl struct {
+}
+
+func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) {
+}
+
+func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) {
+}
+
+var _ InstallHooks = InstallHooksImpl{}

configs/configload/loader_install_test.go

@@ -0,0 +1,65 @@
+package configload
+
+import (
+	"path/filepath"
+	"testing"
+
+	version "github.com/hashicorp/go-version"
+)
+
+func TestLoaderInstallModules_local(t *testing.T) {
+	fixtureDir := filepath.Clean("test-fixtures/local-modules")
+	loader := newTestLoader(filepath.Join(fixtureDir, ".terraform/modules"))
+
+	hooks := &testInstallHooks{}
+
+	diags := loader.InstallModules(fixtureDir, false, hooks)
+	assertNoDiagnostics(t, diags)
+
+	wantCalls := []testInstallHookCall{
+		{
+			Name:        "Install",
+			ModuleAddr:  "child_a",
+			PackageAddr: "",
+			LocalPath:   "test-fixtures/local-modules/child_a",
+		},
+		{
+			Name:        "Install",
+			ModuleAddr:  "child_a.child_b",
+			PackageAddr: "",
+			LocalPath:   "test-fixtures/local-modules/child_a/child_b",
+		},
+	}
+
+	assertResultDeepEqual(t, hooks.Calls, wantCalls)
+}
+
+type testInstallHooks struct {
+	Calls []testInstallHookCall
+}
+
+type testInstallHookCall struct {
+	Name        string
+	ModuleAddr  string
+	PackageAddr string
+	Version     *version.Version
+	LocalPath   string
+}
+
+func (h *testInstallHooks) Download(moduleAddr, packageAddr string, version *version.Version) {
+	h.Calls = append(h.Calls, testInstallHookCall{
+		Name:        "Download",
+		ModuleAddr:  moduleAddr,
+		PackageAddr: packageAddr,
+		Version:     version,
+	})
+}
+
+func (h *testInstallHooks) Install(moduleAddr string, version *version.Version, localPath string) {
+	h.Calls = append(h.Calls, testInstallHookCall{
+		Name:       "Install",
+		ModuleAddr: moduleAddr,
+		Version:    version,
+		LocalPath:  localPath,
+	})
+}

configs/configload/loader_test.go

@@ -4,11 +4,42 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/spf13/afero"
+
 	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/registry"
 	"github.com/zclconf/go-cty/cty"
 )
 
+// newTestLoader is like NewLoader but it uses a copy-on-write overlay filesystem
+// over the real filesystem so that any files that are created cannot persist
+// between test runs.
+//
+// It will also panic if there are any errors creating the loader, since
+// these should never happen in a testing scenario.
+func newTestLoader(dir string) *Loader {
+	realFS := afero.NewOsFs()
+	overlayFS := afero.NewMemMapFs()
+	fs := afero.NewCopyOnWriteFs(realFS, overlayFS)
+	parser := configs.NewParser(fs)
+	reg := registry.NewClient(nil, nil, nil)
+	ret := &Loader{
+		parser: parser,
+		modules: moduleMgr{
+			FS:       afero.Afero{fs},
+			Dir:      dir,
+			Registry: reg,
+		},
+	}
+	err := ret.modules.readModuleManifestSnapshot()
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}
+
 func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool {
 	t.Helper()
 	return assertDiagnosticCount(t, diags, 0)

configs/configload/module_manifest.go

@@ -30,7 +30,7 @@ type moduleRecord struct {
 	// VersionStr is the version specifier string. This is used only for
 	// serialization in snapshots and should not be accessed or updated
 	// by any other codepaths; use "Version" instead.
-	VersionStr string `json:"Version"`
+	VersionStr string `json:"Version,omitempty"`
 
 	// Dir is the path to the local directory where the module is installed.
 	Dir string `json:"Dir"`
@@ -115,7 +115,11 @@ func (m *moduleMgr) writeModuleManifestSnapshot() error {
 	for _, record := range m.manifest {
 		// Make sure VersionStr is in sync with Version, since we encourage
 		// callers to manipulate Version and ignore VersionStr.
-		record.VersionStr = record.Version.String()
+		if record.Version != nil {
+			record.VersionStr = record.Version.String()
+		} else {
+			record.VersionStr = ""
+		}
 		write.Records = append(write.Records, record)
 	}
 

configs/configload/source_addr.go

@@ -0,0 +1,28 @@
+package configload
+
+import (
+	"strings"
+
+	"github.com/hashicorp/terraform/registry/regsrc"
+)
+
+var localSourcePrefixes = []string{
+	"./",
+	"../",
+	".\\",
+	"..\\",
+}
+
+func isLocalSourceAddr(addr string) bool {
+	for _, prefix := range localSourcePrefixes {
+		if strings.HasPrefix(addr, prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func isRegistrySourceAddr(addr string) bool {
+	_, err := regsrc.ParseModuleSource(addr)
+	return err == nil
+}

configs/configload/test-fixtures/local-modules/child_a/child_a.tf

@@ -0,0 +1,4 @@
+
+module "child_b" {
+  source = "./child_b"
+}

configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf

@@ -0,0 +1,4 @@
+
+output "hello" {
+  value = "Hello from child_b!"
+}

configs/configload/test-fixtures/local-modules/root.tf

@@ -0,0 +1,4 @@
+
+module "child_a" {
+  source = "./child_a"
+}