configs/configload: package for loading configurations

Previously the behavior for loading and installing modules was included in
the same package as the representation of the module tree (in the
config/module package).

In our new world, the model of a module tree (now called a "Config") is
included in "configs" along with the Module and File structs. This new
package replaces the loading and installation functionality previously
in config/module with new equivalents that work with the model objects
in "configs".

As of this commit, only the loading functionality is implemented. The
installation functionality will follow in subsequent commits.

configs/config.go

@@ -1,8 +1,6 @@
 package configs
 
 import (
-	"fmt"
-
 	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl2/hcl"
 )
@@ -26,6 +24,17 @@ type Config struct {
 	// this module. If this is the root module then this field is nil.
 	Parent *Config
 
+	// Path is a sequence of module logical names that traverse from the root
+	// module to this config. Path is empty for the root module.
+	//
+	// This should not be used to display a path to the end-user, since
+	// our UI conventions call for us to return a module address string in that
+	// case, and a module address string ought to be built from the dynamic
+	// module tree (resulting from evaluating "count" and "for_each" arguments
+	// on our calls to produce potentially multiple child instances per call)
+	// rather than from our static module tree.
+	Path []string
+
 	// ChildModules points to the Config for each of the direct child modules
 	// called from this module. The keys in this map match the keys in
 	// Module.ModuleCalls.
@@ -64,56 +73,6 @@ type Config struct {
 	Version *version.Version
 }
 
-// Path returns the path of logical names that lead to this Config from its
-// root.
-//
-// This function should not be used to display a path to the end-user, since
-// our UI conventions call for us to return a module address string in that
-// case, and a module address string ought to be built from the dynamic
-// module tree (resulting from evaluating "count" and "for_each" arguments
-// on our calls to produce potentially multiple child instances per call)
-// rather than from our static module tree.
-//
-// This function will panic if called on a config that is not part of a
-// wholesome config tree, e.g. because it has incorrectly-built Children
-// maps, missing node pointers, etc. However, it should work as expected
-// for any tree constructed by BuildConfig and not subsequently modified.
-func (c *Config) Path() []string {
-	// The implementation here is not especially efficient, but we don't
-	// care too much because module trees are shallow and narrow in all
-	// reasonable configurations.
-
-	// We'll build our path in reverse here, since we're starting at the
-	// leafiest node, and then we'll flip it before we return.
-	path := make([]string, 0, c.Depth())
-
-	this := c
-	for this.Parent != nil {
-		parent := this.Parent
-		var name string
-		for candidate, ref := range parent.Children {
-			if ref == this {
-				name = candidate
-			}
-		}
-		if name == "" {
-			panic(fmt.Errorf(
-				"Config %p does not appear in the child table for its parent %p: %#v",
-				this, parent, parent.Children,
-			))
-		}
-		path = append(path, name)
-		this = parent
-	}
-
-	// reverse the items
-	for i := 0; i < len(path)/2; i++ {
-		j := len(path) - i - 1
-		path[i], path[j] = path[j], path[i]
-	}
-	return path
-}
-
 // Depth returns the number of "hops" the receiver is from the root of its
 // module tree, with the root module having a depth of zero.
 func (c *Config) Depth() int {

configs/config_build.go

@@ -29,8 +29,13 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config,
 	calls := parent.Module.ModuleCalls
 
 	for _, call := range calls {
+		path := make([]string, len(parent.Path)+1)
+		copy(path, parent.Path)
+		path[len(path)-1] = call.Name
+
 		req := ModuleRequest{
 			Name:              call.Name,
+			Path:              path,
 			SourceAddr:        call.SourceAddr,
 			SourceAddrRange:   call.SourceAddrRange,
 			VersionConstraint: call.Version,
@@ -50,6 +55,7 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config,
 		child := &Config{
 			Parent:          parent,
 			Root:            parent.Root,
+			Path:            path,
 			Module:          mod,
 			CallRange:       call.DeclRange,
 			SourceAddr:      call.SourceAddr,
@@ -102,6 +108,12 @@ type ModuleRequest struct {
 	// HCL identifier and UTF-8 encoded.
 	Name string
 
+	// Path is a list of logical names that traverse from the root module to
+	// this module. This can be used, for example, to form a lookup key for
+	// each distinct module call in a configuration, allowing for multiple
+	// calls with the same name at different points in the tree.
+	Path []string
+
 	// SourceAddr is the source address string provided by the user in
 	// configuration.
 	SourceAddr string

configs/config_build_test.go

@@ -44,7 +44,7 @@ func TestBuildConfig(t *testing.T) {
 
 	var got []string
 	cfg.DeepEach(func(c *Config) {
-		got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path(), "."), c.Version))
+		got = append(got, fmt.Sprintf("%s %s", strings.Join(c.Path, "."), c.Version))
 	})
 	sort.Strings(got)
 	want := []string{

configs/configload/doc.go

@@ -0,0 +1,4 @@
+// Package configload knows how to install modules into the .terraform/modules
+// directory and to load modules from those installed locations. It is used
+// in conjunction with the LoadConfig function in the parent package.
+package configload

configs/configload/loader.go

@@ -0,0 +1,91 @@
+package configload
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/terraform/configs"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/svchost/auth"
+	"github.com/hashicorp/terraform/svchost/disco"
+	"github.com/spf13/afero"
+)
+
+// A Loader instance is the main entry-point for loading configurations via
+// this package.
+//
+// It extends the general config-loading functionality in the parent package
+// "configs" to support installation of modules from remote sources and
+// loading full configurations using modules that were previously installed.
+type Loader struct {
+	// parser is used to read configuration
+	parser *configs.Parser
+
+	// modules is used to install and locate descendent modules that are
+	// referenced (directly or indirectly) from the root module.
+	modules moduleMgr
+}
+
+// Config is used with NewLoader to specify configuration arguments for the
+// loader.
+type Config struct {
+	// ModulesDir is a path to a directory where descendent modules are
+	// (or should be) installed. (This is usually the
+	// .terraform/modules directory, in the common case where this package
+	// is being loaded from the main Terraform CLI package.)
+	ModulesDir string
+
+	// Services is the service discovery client to use when locating remote
+	// module registry endpoints. If this is nil then registry sources are
+	// not supported, which should be true only in specialized circumstances
+	// such as in tests.
+	Services *disco.Disco
+
+	// Creds is a credentials store for communicating with remote module
+	// registry endpoints. If this is nil then no credentials will be used.
+	Creds auth.CredentialsSource
+}
+
+// NewLoader creates and returns a loader that reads configuration from the
+// real OS filesystem.
+//
+// The loader has some internal state about the modules that are currently
+// installed, which is read from disk as part of this function. If that
+// manifest cannot be read then an error will be returned.
+func NewLoader(config *Config) (*Loader, error) {
+	fs := afero.NewOsFs()
+	parser := configs.NewParser(fs)
+	reg := registry.NewClient(config.Services, config.Creds, nil)
+
+	ret := &Loader{
+		parser: parser,
+		modules: moduleMgr{
+			FS:       afero.Afero{fs},
+			Dir:      config.ModulesDir,
+			Services: config.Services,
+			Creds:    config.Creds,
+			Registry: reg,
+		},
+	}
+
+	err := ret.modules.readModuleManifestSnapshot()
+	if err != nil {
+		return nil, fmt.Errorf("failed to read module manifest: %s", err)
+	}
+
+	return ret, nil
+}
+
+// Parser returns the underlying parser for this loader.
+//
+// This is useful for loading other sorts of files than the module directories
+// that a loader deals with, since then they will share the source code cache
+// for this loader and can thus be shown as snippets in diagnostic messages.
+func (l *Loader) Parser() *configs.Parser {
+	return l.parser
+}
+
+// Sources returns the source code cache for the underlying parser of this
+// loader. This is a shorthand for l.Parser().Sources().
+func (l *Loader) Sources() map[string][]byte {
+	return l.parser.Sources()
+}

configs/configload/loader_load.go

@@ -0,0 +1,97 @@
+package configload
+
+import (
+	"fmt"
+
+	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform/configs"
+)
+
+// LoadConfig reads the Terraform module in the given directory and uses it as the
+// root module to build the static module tree that represents a configuration,
+// assuming that all required descendent modules have already been installed.
+//
+// If error diagnostics are returned, the returned configuration may be either
+// nil or incomplete. In the latter case, cautious static analysis is possible
+// in spite of the errors.
+//
+// LoadConfig performs the basic syntax and uniqueness validations that are
+// required to process the individual modules, and also detects
+func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) {
+	rootMod, diags := l.parser.LoadConfigDir(rootDir)
+	if rootMod == nil {
+		return nil, diags
+	}
+
+	cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad))
+	diags = append(diags, cDiags...)
+
+	return cfg, diags
+}
+
+// moduleWalkerLoad is a configs.ModuleWalkerFunc for loading modules that
+// are presumed to have already been installed. A different function
+// (moduleWalkerInstall) is used for installation.
+func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
+	// Since we're just loading here, we expect that all referenced modules
+	// will be already installed and described in our manifest. However, we
+	// do verify that the manifest and the configuration are in agreement
+	// so that we can prompt the user to run "terraform init" if not.
+
+	key := manifestKey(req.Path)
+	record, exists := l.modules.manifest[key]
+
+	if !exists {
+		return nil, nil, hcl.Diagnostics{
+			{
+				Severity: hcl.DiagError,
+				Summary:  "Module not installed",
+				Detail:   "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.",
+				Subject:  &req.CallRange,
+			},
+		}
+	}
+
+	var diags hcl.Diagnostics
+
+	// Check for inconsistencies between manifest and config
+	if req.SourceAddr != record.SourceAddr {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Module source has changed",
+			Detail:   "The source address was changed since this module was installed. Run \"terraform init\" to install all modules required by this configuration.",
+			Subject:  &req.SourceAddrRange,
+		})
+	}
+	if !req.VersionConstraint.Required.Check(record.Version) {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Module version requirements have changed",
+			Detail: fmt.Sprintf(
+				"The version requirements have changed since this module was installed and the installed version (%s) is no longer acceptable. Run \"terraform init\" to install all modules required by this configuration.",
+				record.Version,
+			),
+			Subject: &req.SourceAddrRange,
+		})
+	}
+
+	mod, mDiags := l.parser.LoadConfigDir(record.Dir)
+	diags = append(diags, mDiags...)
+	if mod == nil {
+		// nil specifically indicates that the directory does not exist or
+		// cannot be read, so in this case we'll discard any generic diagnostics
+		// returned from LoadConfigDir and produce our own context-sensitive
+		// error message.
+		return nil, nil, hcl.Diagnostics{
+			{
+				Severity: hcl.DiagError,
+				Summary:  "Module not installed",
+				Detail:   fmt.Sprintf("This module's local cache directory %s could not be read. Run \"terraform init\" to install all modules required by this configuration.", record.Dir),
+				Subject:  &req.CallRange,
+			},
+		}
+	}
+
+	return mod, record.Version, diags
+}

configs/configload/loader_load_test.go

@@ -0,0 +1,60 @@
+package configload
+
+import (
+	"path/filepath"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/hashicorp/terraform/configs"
+)
+
+func TestLoaderLoadConfig_okay(t *testing.T) {
+	fixtureDir := filepath.Clean("test-fixtures/already-installed")
+	loader, err := NewLoader(&Config{
+		ModulesDir: filepath.Join(fixtureDir, ".terraform/modules"),
+	})
+	if err != nil {
+		t.Fatalf("unexpected error from NewLoader: %s", err)
+	}
+
+	cfg, diags := loader.LoadConfig(fixtureDir)
+	assertNoDiagnostics(t, diags)
+	if cfg == nil {
+		t.Fatalf("config is nil; want non-nil")
+	}
+
+	var gotPaths []string
+	cfg.DeepEach(func(c *configs.Config) {
+		gotPaths = append(gotPaths, strings.Join(c.Path, "."))
+	})
+	sort.Strings(gotPaths)
+	wantPaths := []string{
+		"", // root module
+		"child_a",
+		"child_a.child_c",
+		"child_b",
+		"child_b.child_d",
+	}
+
+	if !reflect.DeepEqual(gotPaths, wantPaths) {
+		t.Fatalf("wrong module paths\ngot: %swant %s", spew.Sdump(gotPaths), spew.Sdump(wantPaths))
+	}
+
+	t.Run("child_a.child_c output", func(t *testing.T) {
+		output := cfg.Children["child_a"].Children["child_c"].Module.Outputs["hello"]
+		got, diags := output.Expr.Value(nil)
+		assertNoDiagnostics(t, diags)
+		assertResultCtyEqual(t, got, cty.StringVal("Hello from child_c"))
+	})
+	t.Run("child_b.child_d output", func(t *testing.T) {
+		output := cfg.Children["child_b"].Children["child_d"].Module.Outputs["hello"]
+		got, diags := output.Expr.Value(nil)
+		assertNoDiagnostics(t, diags)
+		assertResultCtyEqual(t, got, cty.StringVal("Hello from child_d"))
+	})
+}

configs/configload/loader_test.go

@@ -0,0 +1,61 @@
+package configload
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool {
+	t.Helper()
+	return assertDiagnosticCount(t, diags, 0)
+}
+
+func assertDiagnosticCount(t *testing.T, diags hcl.Diagnostics, want int) bool {
+	t.Helper()
+	if len(diags) != 0 {
+		t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want)
+		for _, diag := range diags {
+			t.Logf("- %s", diag)
+		}
+		return true
+	}
+	return false
+}
+
+func assertDiagnosticSummary(t *testing.T, diags hcl.Diagnostics, want string) bool {
+	t.Helper()
+
+	for _, diag := range diags {
+		if diag.Summary == want {
+			return false
+		}
+	}
+
+	t.Errorf("missing diagnostic summary %q", want)
+	for _, diag := range diags {
+		t.Logf("- %s", diag)
+	}
+	return true
+}
+
+func assertResultDeepEqual(t *testing.T, got, want interface{}) bool {
+	t.Helper()
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("wrong result\ngot: %swant: %s", spew.Sdump(got), spew.Sdump(want))
+		return true
+	}
+	return false
+}
+
+func assertResultCtyEqual(t *testing.T, got, want cty.Value) bool {
+	t.Helper()
+	if !got.RawEquals(want) {
+		t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, want)
+		return true
+	}
+	return false
+}

configs/configload/module_manifest.go

@@ -0,0 +1,128 @@
+package configload
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	version "github.com/hashicorp/go-version"
+)
+
+// moduleRecord represents some metadata about an installed module, as part
+// of a moduleManifest.
+type moduleRecord struct {
+	// Key is a unique identifier for this particular module, based on its
+	// position within the static module tree.
+	Key string `json:"Key"`
+
+	// SourceAddr is the source address given for this module in configuration.
+	// This is used only to detect if the source was changed in configuration
+	// since the module was last installed, which means that the installer
+	// must re-install it.
+	SourceAddr string `json:"Source"`
+
+	// Version is the exact version of the module, which results from parsing
+	// VersionStr. nil for un-versioned modules.
+	Version *version.Version `json:"-"`
+
+	// VersionStr is the version specifier string. This is used only for
+	// serialization in snapshots and should not be accessed or updated
+	// by any other codepaths; use "Version" instead.
+	VersionStr string `json:"Version"`
+
+	// Dir is the path to the local directory where the module is installed.
+	Dir string `json:"Dir"`
+}
+
+// moduleManifest is a map used to keep track of the filesystem locations
+// and other metadata about installed modules.
+//
+// The configuration loader refers to this, while the module installer updates
+// it to reflect any changes to the installed modules.
+type moduleManifest map[string]moduleRecord
+
+func manifestKey(path []string) string {
+	return strings.Join(path, ".")
+}
+
+// manifestSnapshotFile is an internal struct used only to assist in our JSON
+// serializtion of manifest snapshots. It should not be used for any other
+// purposes.
+type manifestSnapshotFile struct {
+	Records []moduleRecord `json:"Modules"`
+}
+
+const manifestFilename = "modules.json"
+
+func (m *moduleMgr) manifestSnapshotPath() string {
+	return filepath.Join(m.Dir, manifestFilename)
+}
+
+// readModuleManifestSnapshot loads a manifest snapshot from the filesystem.
+func (m *moduleMgr) readModuleManifestSnapshot() error {
+	src, err := m.FS.ReadFile(m.manifestSnapshotPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// We'll treat a missing file as an empty manifest
+			m.manifest = make(moduleManifest)
+			return nil
+		}
+		return err
+	}
+
+	if len(src) == 0 {
+		// This should never happen, but we'll tolerate it as if it were
+		// a valid empty JSON object.
+		m.manifest = make(moduleManifest)
+		return nil
+	}
+
+	var read manifestSnapshotFile
+	err = json.Unmarshal(src, &read)
+
+	new := make(moduleManifest)
+	for _, record := range read.Records {
+		if record.VersionStr != "" {
+			record.Version, err = version.NewVersion(record.VersionStr)
+			if err != nil {
+				return fmt.Errorf("invalid version %q for %s: %s", record.VersionStr, record.Key, err)
+			}
+		}
+		if _, exists := new[record.Key]; exists {
+			// This should never happen in any valid file, so we'll catch it
+			// and report it to avoid confusing/undefined behavior if the
+			// snapshot file was edited incorrectly outside of Terraform.
+			return fmt.Errorf("snapshot file contains two records for path %s", record.Key)
+		}
+		new[record.Key] = record
+	}
+
+	m.manifest = new
+
+	return nil
+}
+
+// writeModuleManifestSnapshot writes a snapshot of the current manifest
+// to the filesystem.
+//
+// The caller must guarantee no concurrent modifications of the manifest for
+// the duration of a call to this function, or the behavior is undefined.
+func (m *moduleMgr) writeModuleManifestSnapshot() error {
+	var write manifestSnapshotFile
+
+	for _, record := range m.manifest {
+		// Make sure VersionStr is in sync with Version, since we encourage
+		// callers to manipulate Version and ignore VersionStr.
+		record.VersionStr = record.Version.String()
+		write.Records = append(write.Records, record)
+	}
+
+	src, err := json.Marshal(write)
+	if err != nil {
+		return err
+	}
+
+	return m.FS.WriteFile(m.manifestSnapshotPath(), src, os.ModePerm)
+}

configs/configload/module_mgr.go

@@ -0,0 +1,35 @@
+package configload
+
+import (
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/svchost/auth"
+	"github.com/hashicorp/terraform/svchost/disco"
+	"github.com/spf13/afero"
+)
+
+type moduleMgr struct {
+	FS afero.Afero
+
+	// Dir is the path where descendent modules are (or will be) installed.
+	Dir string
+
+	// Services is a service discovery client that will be used to find
+	// remote module registry endpoints. This object may be pre-loaded with
+	// cached discovery information.
+	Services *disco.Disco
+
+	// Creds provides optional credentials for communicating with service hosts.
+	Creds auth.CredentialsSource
+
+	// Registry is a client for the module registry protocol, which is used
+	// when a module is requested from a registry source.
+	Registry *registry.Client
+
+	// manifest tracks the currently-installed modules for this manager.
+	//
+	// The loader may read this. Only the installer may write to it, and
+	// after a set of updates are completed the installer must call
+	// writeModuleManifestSnapshot to persist a snapshot of the manifest
+	// to disk for use on subsequent runs.
+	manifest moduleManifest
+}

configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_a.tf

@@ -0,0 +1,4 @@
+
+module "child_c" {
+  source  = "./child_c"
+}

configs/configload/test-fixtures/already-installed/.terraform/modules/child_a/child_c/child_c.tf

@@ -0,0 +1,4 @@
+
+output "hello" {
+  value = "Hello from child_c"
+}

configs/configload/test-fixtures/already-installed/.terraform/modules/child_b.child_d/child_d.tf

@@ -0,0 +1,4 @@
+
+output "hello" {
+  value = "Hello from child_d"
+}

configs/configload/test-fixtures/already-installed/.terraform/modules/child_b/child_b.tf

@@ -0,0 +1,5 @@
+
+module "child_d" {
+  source  = "example.com/foo/bar_d/baz"
+  # Intentionally no version here
+}

configs/configload/test-fixtures/already-installed/.terraform/modules/modules.json

@@ -0,0 +1 @@
+{"Modules":[{"Key":"","Source":"","Dir":"test-fixtures/already-installed"},{"Key":"child_a","Source":"example.com/foo/bar_a/baz","Version":"1.0.1","Dir":"test-fixtures/already-installed/.terraform/modules/child_a"},{"Key":"child_b","Source":"example.com/foo/bar_b/baz","Version":"1.0.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b"},{"Key":"child_a.child_c","Source":"./child_c","Dir":"test-fixtures/already-installed/.terraform/modules/child_a/child_c"},{"Key":"child_b.child_d","Source":"example.com/foo/bar_d/baz","Version":"1.2.0","Dir":"test-fixtures/already-installed/.terraform/modules/child_b.child_d"}]}

configs/configload/test-fixtures/already-installed/root.tf

@@ -0,0 +1,10 @@
+
+module "child_a" {
+  source  = "example.com/foo/bar_a/baz"
+  version = ">= 1.0.0"
+}
+
+module "child_b" {
+  source = "example.com/foo/bar_b/baz"
+  version = ">= 1.0.0"
+}