terraform: HCL2-flavored module dependency resolver

For the moment this is just a lightly-adapted copy of
ModuleTreeDependencies named ConfigTreeDependencies, with the goal that
the two can live concurrently for the moment while not all callers are yet
updated and then we can drop ModuleTreeDependencies and its helper
functions altogether in a later commit.

This can then be used to make "terraform init" and "terraform providers"
work properly with the HCL2-powered configuration loader.

command/init.go

@@ -7,7 +7,6 @@ import (
 	"sort"
 	"strings"
 
-	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/hcl2/hcl"
 	"github.com/hashicorp/terraform/backend"
 	backendinit "github.com/hashicorp/terraform/backend/init"
@@ -145,6 +144,8 @@ func (c *InitCommand) Run(args []string) int {
 			c.showDiagnostics(diags)
 			return 1
 		}
+
+		c.Ui.Output("")
 	}
 
 	// If our directory is empty, then we're done. We can't get or setup
@@ -289,10 +290,10 @@ func (c *InitCommand) Run(args []string) int {
 	}
 
 	// Now that we have loaded all modules, check the module tree for missing providers.
-	err = c.getProviders(path, state, flagUpgrade)
+	providerDiags := c.getProviders(path, state, flagUpgrade)
-	if err != nil {
+	diags = diags.Append(providerDiags)
-		// this function provides its own output
+	if providerDiags.HasErrors() {
-		log.Printf("[ERROR] %s", err)
+		c.showDiagnostics(diags)
 		return 1
 	}
 
@@ -302,6 +303,11 @@ func (c *InitCommand) Run(args []string) int {
 		c.Ui.Output("")
 	}
 
+	// If we accumulated any warnings along the way that weren't accompanied
+	// by errors then we'll output them here so that the success message is
+	// still the final thing shown.
+	c.showDiagnostics(diags)
+
 	c.Ui.Output(c.Colorize().Color(strings.TrimSpace(outputInitSuccess)))
 	if !c.RunningInAutomation {
 		// If we're not running in an automation wrapper, give the user
@@ -384,24 +390,25 @@ func (c *InitCommand) backendConfigOverrideBody(flags rawFlags, schema *configsc
 
 // Load the complete module tree, and fetch any missing providers.
 // This method outputs its own Ui.
-func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error {
+func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) tfdiags.Diagnostics {
-	mod, diags := c.Module(path)
+	config, diags := c.loadConfig(path)
 	if diags.HasErrors() {
-		c.showDiagnostics(diags)
+		return diags
-		return diags.Err()
 	}
 
 	if err := terraform.CheckStateVersion(state); err != nil {
 		diags = diags.Append(err)
-		c.showDiagnostics(diags)
+		return diags
-		return err
 	}
 
+	// FIXME: Restore this once terraform.CheckRequiredVersion is updated to
+	// work with a configs.Config instead of a legacy module.Tree.
+	/*
 		if err := terraform.CheckRequiredVersion(mod); err != nil {
 			diags = diags.Append(err)
-		c.showDiagnostics(diags)
+			return diags
-		return err
 		}
+	*/
 
 	var available discovery.PluginMetaSet
 	if upgrade {
@@ -412,7 +419,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 		available = c.providerPluginSet()
 	}
 
-	requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
+	requirements := terraform.ConfigTreeDependencies(config, state).AllPluginRequirements()
 	if len(requirements) == 0 {
 		// nothing to initialize
 		return nil
@@ -424,7 +431,6 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 
 	missing := c.missingPlugins(available, requirements)
 
-	var errs error
 	if c.getPlugins {
 		if len(missing) > 0 {
 			c.Ui.Output(fmt.Sprintf("- Checking for available provider plugins on %s...",
@@ -461,12 +467,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 					c.Ui.Error(fmt.Sprintf(errProviderInstallError, provider, err.Error(), DefaultPluginVendorDir))
 				}
 
-				errs = multierror.Append(errs, err)
+				diags = diags.Append(err)
 			}
 		}
 
-		if errs != nil {
+		if diags.HasErrors() {
-			return errs
+			return diags
 		}
 	} else if len(missing) > 0 {
 		// we have missing providers, but aren't going to try and download them
@@ -477,11 +483,11 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 			} else {
 				lines = append(lines, fmt.Sprintf("* %s (%s)\n", provider, reqd.Versions))
 			}
-			errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider))
+			diags = diags.Append(fmt.Errorf("missing provider %q", provider))
 		}
 		sort.Strings(lines)
 		c.Ui.Error(fmt.Sprintf(errMissingProvidersNoInstall, strings.Join(lines, ""), DefaultPluginVendorDir))
-		return errs
+		return diags
 	}
 
 	// With all the providers downloaded, we'll generate our lock file
@@ -497,8 +503,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 	for name, meta := range chosen {
 		digest, err := meta.SHA256()
 		if err != nil {
-			c.Ui.Error(fmt.Sprintf("failed to read provider plugin %s: %s", meta.Path, err))
+			diags = diags.Append(fmt.Errorf("Failed to read provider plugin %s: %s", meta.Path, err))
-			return err
+			return diags
 		}
 		digests[name] = digest
 		if c.ignorePluginChecksum {
@@ -507,8 +513,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 	}
 	err := c.providerPluginsLock().Write(digests)
 	if err != nil {
-		c.Ui.Error(fmt.Sprintf("failed to save provider manifest: %s", err))
+		diags = diags.Append(fmt.Errorf("failed to save provider manifest: %s", err))
-		return err
+		return diags
 	}
 
 	{
@@ -556,7 +562,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 		}
 	}
 
-	return nil
+	return diags
 }
 
 func (c *InitCommand) AutocompleteArgs() complete.Predictor {

command/providers.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 
 	"github.com/hashicorp/terraform/moduledeps"
+	"github.com/hashicorp/terraform/terraform"
 	"github.com/hashicorp/terraform/tfdiags"
 	"github.com/xlab/treeprint"
 )
@@ -69,11 +70,8 @@ func (c *ProvidersCommand) Run(args []string) int {
 		return 1
 	}
 
-	// FIXME: Restore this once the "terraform" package is updated to deal
+	s := state.State()
-	// with HCL2 config types.
+	depTree := terraform.ConfigTreeDependencies(config, s)
-	//s := state.State()
-	//depTree := terraform.ModuleTreeDependencies(config, s)
-	var depTree *moduledeps.Module
 	depTree.SortDescendents()
 
 	printRoot := treeprint.New()

configs/resource.go

@@ -2,6 +2,7 @@ package configs
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/hashicorp/hcl2/gohcl"
 	"github.com/hashicorp/hcl2/hcl"
@@ -39,6 +40,22 @@ func (r *ManagedResource) moduleUniqueKey() string {
 	return fmt.Sprintf("%s.%s", r.Name, r.Type)
 }
 
+// ProviderConfigKey returns a string key for the provider configuration
+// that should be used for this resource. This function implements the
+// default behavior of extracting the type from the resource type name if
+// an explicit "provider" argument was not provided.
+func (r *ManagedResource) ProviderConfigKey() string {
+	if r.ProviderConfigRef == nil {
+		typeName := r.Type
+		if under := strings.Index(typeName, "_"); under != -1 {
+			return typeName[:under]
+		}
+		return typeName
+	}
+
+	return r.ProviderConfigRef.String()
+}
+
 func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) {
 	r := &ManagedResource{
 		Type:      block.Labels[0],
@@ -234,6 +251,22 @@ func (r *DataResource) moduleUniqueKey() string {
 	return fmt.Sprintf("data.%s.%s", r.Name, r.Type)
 }
 
+// ProviderConfigKey returns a string key for the provider configuration
+// that should be used for this resource. This function implements the
+// default behavior of extracting the type from the resource type name if
+// an explicit "provider" argument was not provided.
+func (r *DataResource) ProviderConfigKey() string {
+	if r.ProviderConfigRef == nil {
+		typeName := r.Type
+		if under := strings.Index(typeName, "_"); under != -1 {
+			return typeName[:under]
+		}
+		return typeName
+	}
+
+	return r.ProviderConfigRef.String()
+}
+
 func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) {
 	r := &DataResource{
 		Type:      block.Labels[0],
@@ -370,6 +403,16 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn
 	return ret, diags
 }
 
+func (r *ProviderConfigRef) String() string {
+	if r == nil {
+		return "<nil>"
+	}
+	if r.Alias != "" {
+		return fmt.Sprintf("%s.%s", r.Name, r.Alias)
+	}
+	return r.Name
+}
+
 var commonResourceAttributes = []hcl.AttributeSchema{
 	{
 		Name: "count",

plugin/discovery/version_set.go

@@ -36,6 +36,11 @@ type Constraints struct {
 	raw version.Constraints
 }
 
+// NewConstraints creates a Constraints based on a version.Constraints.
+func NewConstraints(c version.Constraints) Constraints {
+	return Constraints{c}
+}
+
 // AllVersions is a Constraints containing all versions
 var AllVersions Constraints
 

terraform/module_dependencies.go

@@ -1,12 +1,207 @@
 package terraform
 
 import (
+	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/configs"
 	"github.com/hashicorp/terraform/moduledeps"
 	"github.com/hashicorp/terraform/plugin/discovery"
 )
 
+// ConfigTreeDependencies returns the dependencies of the tree of modules
+// described by the given configuration and state.
+//
+// Both configuration and state are required because there can be resources
+// implied by instances in the state that no longer exist in config.
+func ConfigTreeDependencies(root *configs.Config, state *State) *moduledeps.Module {
+	// First we walk the configuration tree to build the overall structure
+	// and capture the explicit/implicit/inherited provider dependencies.
+	deps := configTreeConfigDependencies(root, nil)
+
+	// Next we walk over the resources in the state to catch any additional
+	// dependencies created by existing resources that are no longer in config.
+	// Most things we find in state will already be present in 'deps', but
+	// we're interested in the rare thing that isn't.
+	configTreeMergeStateDependencies(deps, state)
+
+	return deps
+}
+
+func configTreeConfigDependencies(root *configs.Config, inheritProviders map[string]*configs.Provider) *moduledeps.Module {
+	if root == nil {
+		// If no config is provided, we'll make a synthetic root.
+		// This isn't necessarily correct if we're called with a nil that
+		// *isn't* at the root, but in practice that can never happen.
+		return &moduledeps.Module{
+			Name: "root",
+		}
+	}
+
+	name := "root"
+	if len(root.Path) != 0 {
+		name = root.Path[len(root.Path)-1]
+	}
+
+	ret := &moduledeps.Module{
+		Name: name,
+	}
+
+	module := root.Module
+
+	// Provider dependencies
+	{
+		providers := make(moduledeps.Providers)
+
+		// The main way to declare a provider dependency is explicitly inside
+		// the "terraform" block, which allows declaring a requirement without
+		// also creating a configuration.
+		for fullName, constraints := range module.ProviderRequirements {
+			inst := moduledeps.ProviderInstance(fullName)
+
+			// The handling here is a bit fiddly because the moduledeps package
+			// was designed around the legacy (pre-0.12) configuration model
+			// and hasn't yet been revised to handle the new model. As a result,
+			// we need to do some translation here.
+			// FIXME: Eventually we should adjust the underlying model so we
+			// can also retain the source location of each constraint, for
+			// more informative output from the "terraform providers" command.
+			var rawConstraints version.Constraints
+			for _, constraint := range constraints {
+				rawConstraints = append(rawConstraints, constraint.Required...)
+			}
+			discoConstraints := discovery.NewConstraints(rawConstraints)
+
+			providers[inst] = moduledeps.ProviderDependency{
+				Constraints: discoConstraints,
+				Reason:      moduledeps.ProviderDependencyExplicit,
+			}
+		}
+
+		// Provider configurations can also include version constraints,
+		// allowing for more terse declaration in situations where both a
+		// configuration and a constraint are defined in the same module.
+		for fullName, pCfg := range module.ProviderConfigs {
+			inst := moduledeps.ProviderInstance(fullName)
+			discoConstraints := discovery.AllVersions
+			if pCfg.Version.Required != nil {
+				discoConstraints = discovery.NewConstraints(pCfg.Version.Required)
+			}
+			if existing, exists := providers[inst]; exists {
+				existing.Constraints = existing.Constraints.Append(discoConstraints)
+			} else {
+				providers[inst] = moduledeps.ProviderDependency{
+					Constraints: discoConstraints,
+					Reason:      moduledeps.ProviderDependencyExplicit,
+				}
+			}
+		}
+
+		// Each resource in the configuration creates an *implicit* provider
+		// dependency, though we'll only record it if there isn't already
+		// an explicit dependency on the same provider.
+		for _, rc := range module.ManagedResources {
+			fullName := rc.ProviderConfigKey()
+			inst := moduledeps.ProviderInstance(fullName)
+			if _, exists := providers[inst]; exists {
+				// Explicit dependency already present
+				continue
+			}
+
+			reason := moduledeps.ProviderDependencyImplicit
+			if _, inherited := inheritProviders[fullName]; inherited {
+				reason = moduledeps.ProviderDependencyInherited
+			}
+
+			providers[inst] = moduledeps.ProviderDependency{
+				Constraints: discovery.AllVersions,
+				Reason:      reason,
+			}
+		}
+		for _, rc := range module.DataResources {
+			fullName := rc.ProviderConfigKey()
+			inst := moduledeps.ProviderInstance(fullName)
+			if _, exists := providers[inst]; exists {
+				// Explicit dependency already present
+				continue
+			}
+
+			reason := moduledeps.ProviderDependencyImplicit
+			if _, inherited := inheritProviders[fullName]; inherited {
+				reason = moduledeps.ProviderDependencyInherited
+			}
+
+			providers[inst] = moduledeps.ProviderDependency{
+				Constraints: discovery.AllVersions,
+				Reason:      reason,
+			}
+		}
+
+		ret.Providers = providers
+	}
+
+	childInherit := make(map[string]*configs.Provider)
+	for k, v := range inheritProviders {
+		childInherit[k] = v
+	}
+	for k, v := range module.ProviderConfigs {
+		childInherit[k] = v
+	}
+	for _, c := range root.Children {
+		ret.Children = append(ret.Children, configTreeConfigDependencies(c, childInherit))
+	}
+
+	return ret
+}
+
+func configTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
+	if state == nil {
+		return
+	}
+
+	findModule := func(path []string) *moduledeps.Module {
+		module := root
+		for _, name := range path[1:] { // skip initial "root"
+			var next *moduledeps.Module
+			for _, cm := range module.Children {
+				if cm.Name == name {
+					next = cm
+					break
+				}
+			}
+
+			if next == nil {
+				// If we didn't find a next node, we'll need to make one
+				next = &moduledeps.Module{
+					Name: name,
+				}
+				module.Children = append(module.Children, next)
+			}
+
+			module = next
+		}
+		return module
+	}
+
+	for _, ms := range state.Modules {
+		module := findModule(ms.Path)
+
+		for _, is := range ms.Resources {
+			fullName := config.ResourceProviderFullName(is.Type, is.Provider)
+			inst := moduledeps.ProviderInstance(fullName)
+			if _, exists := module.Providers[inst]; !exists {
+				if module.Providers == nil {
+					module.Providers = make(moduledeps.Providers)
+				}
+				module.Providers[inst] = moduledeps.ProviderDependency{
+					Constraints: discovery.AllVersions,
+					Reason:      moduledeps.ProviderDependencyFromState,
+				}
+			}
+		}
+	}
+}
+
 // ModuleTreeDependencies returns the dependencies of the tree of modules
 // described by the given configuration tree and state.
 //
@@ -106,50 +301,9 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string
 }
 
 func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
-	if state == nil {
+	// This is really just the same logic as configTreeMergeStateDependencies
-		return
+	// but we retain this old name just to keep the symmetry until we've
-	}
+	// removed all of these "moduleTree..." versions that use the legacy
-
+	// configuration structs.
-	findModule := func(path []string) *moduledeps.Module {
+	configTreeMergeStateDependencies(root, state)
-		module := root
-		for _, name := range path[1:] { // skip initial "root"
-			var next *moduledeps.Module
-			for _, cm := range module.Children {
-				if cm.Name == name {
-					next = cm
-					break
-				}
-			}
-
-			if next == nil {
-				// If we didn't find a next node, we'll need to make one
-				next = &moduledeps.Module{
-					Name: name,
-				}
-				module.Children = append(module.Children, next)
-			}
-
-			module = next
-		}
-		return module
-	}
-
-	for _, ms := range state.Modules {
-		module := findModule(ms.Path)
-
-		for _, is := range ms.Resources {
-			fullName := config.ResourceProviderFullName(is.Type, is.Provider)
-			inst := moduledeps.ProviderInstance(fullName)
-			if _, exists := module.Providers[inst]; !exists {
-				if module.Providers == nil {
-					module.Providers = make(moduledeps.Providers)
-				}
-				module.Providers[inst] = moduledeps.ProviderDependency{
-					Constraints: discovery.AllVersions,
-					Reason:      moduledeps.ProviderDependencyFromState,
-				}
-			}
-		}
-	}
-
 }