terraform/internal/initwd/module_install.go

package initwd

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"

	version "github.com/hashicorp/go-version"
	"github.com/hashicorp/terraform-config-inspect/tfconfig"
	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/internal/earlyconfig"
	"github.com/hashicorp/terraform/internal/modsdir"
	"github.com/hashicorp/terraform/registry"
	"github.com/hashicorp/terraform/registry/regsrc"
	"github.com/hashicorp/terraform/registry/response"
	"github.com/hashicorp/terraform/tfdiags"
)

type ModuleInstaller struct {
	modsDir string
	reg     *registry.Client

	// The keys in moduleVersions are resolved and trimmed registry source
	// addresses and the values are the registry response.
	moduleVersions map[string]*response.ModuleVersions

	// The keys in moduleVersionsUrl are the moduleVersion struct below and
	// addresses and the values are the download URLs.
	moduleVersionsUrl map[moduleVersion]string
}

type moduleVersion struct {
	module  string
	version string
}

func NewModuleInstaller(modsDir string, reg *registry.Client) *ModuleInstaller {
	return &ModuleInstaller{
		modsDir:           modsDir,
		reg:               reg,
		moduleVersions:    make(map[string]*response.ModuleVersions),
		moduleVersionsUrl: make(map[moduleVersion]string),
	}
}

// InstallModules analyses the root module in the given directory and installs
// all of its direct and transitive dependencies into the given modules
// directory, which must already exist.
//
// Since InstallModules makes possibly-time-consuming calls to remote services,
// a hook interface is supported to allow the caller to be notified when
// each module is installed and, for remote modules, when downloading begins.
// LoadConfig guarantees that two hook calls will not happen concurrently but
// it does not guarantee any particular ordering of hook calls. This mechanism
// is for UI feedback only and does not give the caller any control over the
// process.
//
// If modules are already installed in the target directory, they will be
// skipped unless their source address or version have changed or unless
// the upgrade flag is set.
//
// InstallModules never deletes any directory, except in the case where it
// needs to replace a directory that is already present with a newly-extracted
// package.
//
// If the returned diagnostics contains errors then the module installation
// may have wholly or partially completed. Modules must be loaded in order
// to find their dependencies, so this function does many of the same checks
// as LoadConfig as a side-effect.
//
// If successful (the returned diagnostics contains no errors) then the
// first return value is the early configuration tree that was constructed by
// the installation process.
func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks ModuleInstallHooks) (*earlyconfig.Config, tfdiags.Diagnostics) {
	log.Printf("[TRACE] ModuleInstaller: installing child modules for %s into %s", rootDir, i.modsDir)

	rootMod, diags := earlyconfig.LoadModule(rootDir)
	if rootMod == nil {
		return nil, diags
	}

	manifest, err := modsdir.ReadManifestSnapshotForDir(i.modsDir)
	if err != nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Failed to read modules manifest file",
			fmt.Sprintf("Error reading manifest for %s: %s.", i.modsDir, err),
		))
		return nil, diags
	}

	getter := reusingGetter{}
	cfg, instDiags := i.installDescendentModules(rootMod, rootDir, manifest, upgrade, hooks, getter)
	diags = append(diags, instDiags...)

	return cfg, diags
}

func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks ModuleInstallHooks, getter reusingGetter) (*earlyconfig.Config, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	if hooks == nil {
		// Use our no-op implementation as a placeholder
		hooks = ModuleInstallHooksImpl{}
	}

	// Create a manifest record for the root module. This will be used if
	// there are any relative-pathed modules in the root.
	manifest[""] = modsdir.Record{
		Key: "",
		Dir: rootDir,
	}

	cfg, cDiags := earlyconfig.BuildConfig(rootMod, earlyconfig.ModuleWalkerFunc(
		func(req *earlyconfig.ModuleRequest) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {

			key := manifest.ModuleKey(req.Path)
			instPath := i.packageInstallPath(req.Path)

			log.Printf("[DEBUG] Module installer: begin %s", key)

			// First we'll check if we need to upgrade/replace an existing
			// installed module, and delete it out of the way if so.
			replace := upgrade
			if !replace {
				record, recorded := manifest[key]
				switch {
				case !recorded:
					log.Printf("[TRACE] ModuleInstaller: %s is not yet installed", key)
					replace = true
				case record.SourceAddr != req.SourceAddr:
					log.Printf("[TRACE] ModuleInstaller: %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
					replace = true
				case record.Version != nil && !req.VersionConstraints.Check(record.Version):
					log.Printf("[TRACE] ModuleInstaller: %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints)
					replace = true
				}
			}

			// If we _are_ planning to replace this module, then we'll remove
			// it now so our installation code below won't conflict with any
			// existing remnants.
			if replace {
				if _, recorded := manifest[key]; recorded {
					log.Printf("[TRACE] ModuleInstaller: discarding previous record of %s prior to reinstall", key)
				}
				delete(manifest, key)
				// Deleting a module invalidates all of its descendent modules too.
				keyPrefix := key + "."
				for subKey := range manifest {
					if strings.HasPrefix(subKey, keyPrefix) {
						if _, recorded := manifest[subKey]; recorded {
							log.Printf("[TRACE] ModuleInstaller: also discarding downstream %s", subKey)
						}
						delete(manifest, subKey)
					}
				}
			}

			record, recorded := manifest[key]
			if !recorded {
				// Clean up any stale cache directory that might be present.
				// If this is a local (relative) source then the dir will
				// not exist, but we'll ignore that.
				log.Printf("[TRACE] ModuleInstaller: cleaning directory %s prior to install of %s", instPath, key)
				err := os.RemoveAll(instPath)
				if err != nil && !os.IsNotExist(err) {
					log.Printf("[TRACE] ModuleInstaller: failed to remove %s: %s", key, err)
					diags = diags.Append(tfdiags.Sourceless(
						tfdiags.Error,
						"Failed to remove local module cache",
						fmt.Sprintf(
							"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
							instPath, err,
						),
					))
					return nil, nil, diags
				}
			} else {
				// If this module is already recorded and its root directory
				// exists then we will just load what's already there and
				// keep our existing record.
				info, err := os.Stat(record.Dir)
				if err == nil && info.IsDir() {
					mod, mDiags := earlyconfig.LoadModule(record.Dir)
					diags = diags.Append(mDiags)

					log.Printf("[TRACE] ModuleInstaller: Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
					return mod, record.Version, diags
				}
			}

			// If we get down here then it's finally time to actually install
			// the module. There are some variants to this process depending
			// on what type of module source address we have.
			switch {

			case isLocalSourceAddr(req.SourceAddr):
				log.Printf("[TRACE] ModuleInstaller: %s has local path %q", key, req.SourceAddr)
				mod, mDiags := i.installLocalModule(req, key, manifest, hooks)
				diags = append(diags, mDiags...)
				return mod, nil, diags

			case isRegistrySourceAddr(req.SourceAddr):
				addr, err := regsrc.ParseModuleSource(req.SourceAddr)
				if err != nil {
					// Should never happen because isRegistrySourceAddr already validated
					panic(err)
				}
				log.Printf("[TRACE] ModuleInstaller: %s is a registry module at %s", key, addr)

				mod, v, mDiags := i.installRegistryModule(req, key, instPath, addr, manifest, hooks, getter)
				diags = append(diags, mDiags...)
				return mod, v, diags

			default:
				log.Printf("[TRACE] ModuleInstaller: %s address %q will be handled by go-getter", key, req.SourceAddr)

				mod, mDiags := i.installGoGetterModule(req, key, instPath, manifest, hooks, getter)
				diags = append(diags, mDiags...)
				return mod, nil, diags
			}

		},
	))
	diags = append(diags, cDiags...)

	err := manifest.WriteSnapshotToDir(i.modsDir)
	if err != nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Failed to update module manifest",
			fmt.Sprintf("Unable to write the module manifest file: %s", err),
		))
	}

	return cfg, diags
}

func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks ModuleInstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	parentKey := manifest.ModuleKey(req.Parent.Path)
	parentRecord, recorded := manifest[parentKey]
	if !recorded {
		// This is indicative of a bug rather than a user-actionable error
		panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
	}

	if len(req.VersionConstraints) != 0 {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Invalid version constraint",
			fmt.Sprintf("Cannot apply a version constraint to module %q (at %s:%d) because it has a relative local path.", req.Name, req.CallPos.Filename, req.CallPos.Line),
		))
	}

	// For local sources we don't actually need to modify the
	// filesystem at all because the parent already wrote
	// the files we need, and so we just load up what's already here.
	newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)

	log.Printf("[TRACE] ModuleInstaller: %s uses directory from parent: %s", key, newDir)
	// it is possible that the local directory is a symlink
	newDir, err := filepath.EvalSymlinks(newDir)
	if err != nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Unreadable module directory",
			fmt.Sprintf("Unable to evaluate directory symlink: %s", err.Error()),
		))
	}

	mod, mDiags := earlyconfig.LoadModule(newDir)
	if mod == nil {
		// nil indicates missing or unreadable directory, so we'll
		// discard the returned diags and return a more specific
		// error message here.
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Unreadable module directory",
			fmt.Sprintf("The directory %s could not be read for module %q at %s:%d.", newDir, req.Name, req.CallPos.Filename, req.CallPos.Line),
		))
	} else {
		diags = diags.Append(mDiags)
	}

	// Note the local location in our manifest.
	manifest[key] = modsdir.Record{
		Key:        key,
		Dir:        newDir,
		SourceAddr: req.SourceAddr,
	}
	log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
	hooks.Install(key, nil, newDir)

	return mod, diags
}

func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	hostname, err := addr.SvcHost()
	if err != nil {
		// If it looks like the user was trying to use punycode then we'll generate
		// a specialized error for that case. We require the unicode form of
		// hostname so that hostnames are always human-readable in configuration
		// and punycode can't be used to hide a malicious module hostname.
		if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid module registry hostname",
				fmt.Sprintf("The hostname portion of the module %q source address (at %s:%d) is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.", req.Name, req.CallPos.Filename, req.CallPos.Line),
			))
		} else {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Invalid module registry hostname",
				fmt.Sprintf("The hostname portion of the module %q source address (at %s:%d) is not a valid hostname.", req.Name, req.CallPos.Filename, req.CallPos.Line),
			))
		}
		return nil, nil, diags
	}

	reg := i.reg
	var resp *response.ModuleVersions
	var exists bool

	// check if we've already looked up this module from the registry
	if resp, exists = i.moduleVersions[addr.String()]; exists {
		log.Printf("[TRACE] %s using already found available versions of %s at %s", key, addr, hostname)
	} else {
		log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
		resp, err = reg.ModuleVersions(addr)
		if err != nil {
			if registry.IsModuleNotFound(err) {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Module not found",
					fmt.Sprintf("Module %q (from %s:%d) cannot be found in the module registry at %s.", req.Name, req.CallPos.Filename, req.CallPos.Line, hostname),
				))
			} else {
				diags = diags.Append(tfdiags.Sourceless(
					tfdiags.Error,
					"Error accessing remote module registry",
					fmt.Sprintf("Failed to retrieve available versions for module %q (%s:%d) from %s: %s.", req.Name, req.CallPos.Filename, req.CallPos.Line, hostname, err),
				))
			}
			return nil, nil, diags
		}
		i.moduleVersions[addr.String()] = resp
	}

	// The response might contain information about dependencies to allow us
	// to potentially optimize future requests, but we don't currently do that
	// and so for now we'll just take the first item which is guaranteed to
	// be the address we requested.
	if len(resp.Modules) < 1 {
		// Should never happen, but since this is a remote service that may
		// be implemented by third-parties we will handle it gracefully.
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Invalid response from remote module registry",
			fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for module %q (%s:%d).", hostname, req.Name, req.CallPos.Filename, req.CallPos.Line),
		))
		return nil, nil, diags
	}

	modMeta := resp.Modules[0]

	var latestMatch *version.Version
	var latestVersion *version.Version
	for _, mv := range modMeta.Versions {
		v, err := version.NewVersion(mv.Version)
		if err != nil {
			// Should never happen if the registry server is compliant with
			// the protocol, but we'll warn if not to assist someone who
			// might be developing a module registry server.
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Warning,
				"Invalid response from remote module registry",
				fmt.Sprintf("The registry at %s returned an invalid version string %q for module %q (%s:%d), which Terraform ignored.", hostname, mv.Version, req.Name, req.CallPos.Filename, req.CallPos.Line),
			))
			continue
		}

		// If we've found a pre-release version then we'll ignore it unless
		// it was exactly requested.
		if v.Prerelease() != "" && req.VersionConstraints.String() != v.String() {
			log.Printf("[TRACE] ModuleInstaller: %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
			continue
		}

		if latestVersion == nil || v.GreaterThan(latestVersion) {
			latestVersion = v
		}

		if req.VersionConstraints.Check(v) {
			if latestMatch == nil || v.GreaterThan(latestMatch) {
				latestMatch = v
			}
		}
	}

	if latestVersion == nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Module has no versions",
			fmt.Sprintf("Module %q (%s:%d) has no versions available on %s.", addr, req.CallPos.Filename, req.CallPos.Line, hostname),
		))
		return nil, nil, diags
	}

	if latestMatch == nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Unresolvable module version constraint",
			fmt.Sprintf("There is no available version of module %q (%s:%d) which matches the given version constraint. The newest available version is %s.", addr, req.CallPos.Filename, req.CallPos.Line, latestVersion),
		))
		return nil, nil, diags
	}

	// Report up to the caller that we're about to start downloading.
	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
	hooks.Download(key, packageAddr, latestMatch)

	// If we manage to get down here then we've found a suitable version to
	// install, so we need to ask the registry where we should download it from.
	// The response to this is a go-getter-style address string.

	// first check the cache for the download URL
	moduleAddr := moduleVersion{module: addr.String(), version: latestMatch.String()}
	if _, exists := i.moduleVersionsUrl[moduleAddr]; !exists {
		url, err := reg.ModuleLocation(addr, latestMatch.String())
		if err != nil {
			log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Error accessing remote module registry",
				fmt.Sprintf("Failed to retrieve a download URL for %s %s from %s: %s", addr, latestMatch, hostname, err),
			))
			return nil, nil, diags
		}
		i.moduleVersionsUrl[moduleVersion{module: addr.String(), version: latestMatch.String()}] = url
	}

	dlAddr := i.moduleVersionsUrl[moduleAddr]

	log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr)

	modDir, err := getter.getWithGoGetter(instPath, dlAddr)
	if err != nil {
		// Errors returned by go-getter have very inconsistent quality as
		// end-user error messages, but for now we're accepting that because
		// we have no way to recognize any specific errors to improve them
		// and masking the error entirely would hide valuable diagnostic
		// information from the user.
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Failed to download module",
			fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s.", req.Name, req.CallPos.Filename, req.CallPos.Line, dlAddr, err),
		))
		return nil, nil, diags
	}

	log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, dlAddr, modDir)

	if addr.RawSubmodule != "" {
		// Append the user's requested subdirectory to any subdirectory that
		// was implied by any of the nested layers we expanded within go-getter.
		modDir = filepath.Join(modDir, addr.RawSubmodule)
	}

	log.Printf("[TRACE] ModuleInstaller: %s should now be at %s", key, modDir)

	// Finally we are ready to try actually loading the module.
	mod, mDiags := earlyconfig.LoadModule(modDir)
	if mod == nil {
		// nil indicates missing or unreadable directory, so we'll
		// discard the returned diags and return a more specific
		// error message here. For registry modules this actually
		// indicates a bug in the code above, since it's not the
		// user's responsibility to create the directory in this case.
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Unreadable module directory",
			fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
		))
	} else {
		diags = append(diags, mDiags...)
	}

	// Note the local location in our manifest.
	manifest[key] = modsdir.Record{
		Key:        key,
		Version:    latestMatch,
		Dir:        modDir,
		SourceAddr: req.SourceAddr,
	}
	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
	hooks.Install(key, latestMatch, modDir)

	return mod, latestMatch, diags
}

func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	// Report up to the caller that we're about to start downloading.
	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
	hooks.Download(key, packageAddr, nil)

	if len(req.VersionConstraints) != 0 {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Invalid version constraint",
			fmt.Sprintf("Cannot apply a version constraint to module %q (at %s:%d) because it has a non Registry URL.", req.Name, req.CallPos.Filename, req.CallPos.Line),
		))
		return nil, diags
	}

	modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr)
	if err != nil {
		if _, ok := err.(*MaybeRelativePathErr); ok {
			log.Printf(
				"[TRACE] ModuleInstaller: %s looks like a local path but is missing ./ or ../",
				req.SourceAddr,
			)
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Module not found",
				fmt.Sprintf(
					"The module address %q could not be resolved.\n\n"+
						"If you intended this as a path relative to the current "+
						"module, use \"./%s\" instead. The \"./\" prefix "+
						"indicates that the address is a relative filesystem path.",
					req.SourceAddr, req.SourceAddr,
				),
			))
		} else {
			// Errors returned by go-getter have very inconsistent quality as
			// end-user error messages, but for now we're accepting that because
			// we have no way to recognize any specific errors to improve them
			// and masking the error entirely would hide valuable diagnostic
			// information from the user.
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Failed to download module",
				fmt.Sprintf("Could not download module %q (%s:%d) source code from %q: %s", req.Name, req.CallPos.Filename, req.CallPos.Line, packageAddr, err),
			))
		}
		return nil, diags

	}

	log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, req.SourceAddr, modDir)

	mod, mDiags := earlyconfig.LoadModule(modDir)
	if mod == nil {
		// nil indicates missing or unreadable directory, so we'll
		// discard the returned diags and return a more specific
		// error message here. For go-getter modules this actually
		// indicates a bug in the code above, since it's not the
		// user's responsibility to create the directory in this case.
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Unreadable module directory",
			fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
		))
	} else {
		diags = append(diags, mDiags...)
	}

	// Note the local location in our manifest.
	manifest[key] = modsdir.Record{
		Key:        key,
		Dir:        modDir,
		SourceAddr: req.SourceAddr,
	}
	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
	hooks.Install(key, nil, modDir)

	return mod, diags
}

func (i *ModuleInstaller) packageInstallPath(modulePath addrs.Module) string {
	return filepath.Join(i.modsDir, strings.Join(modulePath, "."))
}