diff --git a/backend/local/backend_apply_test.go b/backend/local/backend_apply_test.go index 940d08bb0..e7e7a53a9 100644 --- a/backend/local/backend_apply_test.go +++ b/backend/local/backend_apply_test.go @@ -13,8 +13,8 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/hashicorp/terraform/backend" - "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/configs/configschema" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/providers" "github.com/hashicorp/terraform/states" "github.com/hashicorp/terraform/states/statemgr" @@ -252,7 +252,7 @@ func (s failingState) WriteState(state *states.State) error { func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) { t.Helper() - _, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir) + _, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir) return &backend.Operation{ Type: backend.OperationTypeApply, diff --git a/backend/local/backend_plan_test.go b/backend/local/backend_plan_test.go index bd0da1c91..9685ad3df 100644 --- a/backend/local/backend_plan_test.go +++ b/backend/local/backend_plan_test.go @@ -10,8 +10,8 @@ import ( "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/backend" - "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/configs/configschema" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/plans" "github.com/hashicorp/terraform/plans/planfile" "github.com/hashicorp/terraform/states" @@ -514,7 +514,7 @@ func TestLocal_planScaleOutNoDupeCount(t *testing.T) { func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) { t.Helper() - _, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir) + _, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir) return &backend.Operation{ Type: backend.OperationTypePlan, diff --git a/backend/local/backend_refresh_test.go b/backend/local/backend_refresh_test.go index 40e8e906e..c58c0597e 100644 --- a/backend/local/backend_refresh_test.go +++ b/backend/local/backend_refresh_test.go @@ -8,8 +8,8 @@ import ( "github.com/hashicorp/terraform/providers" "github.com/hashicorp/terraform/backend" - "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/configs/configschema" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/terraform" "github.com/zclconf/go-cty/cty" ) @@ -203,7 +203,7 @@ test_instance.foo: func testOperationRefresh(t *testing.T, configDir string) (*backend.Operation, func()) { t.Helper() - _, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir) + _, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir) return &backend.Operation{ Type: backend.OperationTypeRefresh, diff --git a/backend/remote/backend_apply_test.go b/backend/remote/backend_apply_test.go index 5b2698a94..f78d35dd6 100644 --- a/backend/remote/backend_apply_test.go +++ b/backend/remote/backend_apply_test.go @@ -12,7 +12,7 @@ import ( tfe "github.com/hashicorp/go-tfe" "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/backend" - "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/plans/planfile" "github.com/hashicorp/terraform/terraform" "github.com/mitchellh/cli" @@ -21,7 +21,7 @@ import ( func testOperationApply(t *testing.T, configDir string) (*backend.Operation, func()) { t.Helper() - _, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir) + _, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir) return &backend.Operation{ ConfigDir: configDir, diff --git a/backend/remote/backend_plan_test.go b/backend/remote/backend_plan_test.go index 5c96ebdd5..8d1bd0e80 100644 --- a/backend/remote/backend_plan_test.go +++ b/backend/remote/backend_plan_test.go @@ -12,7 +12,7 @@ import ( tfe "github.com/hashicorp/go-tfe" "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/backend" - "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/plans/planfile" "github.com/hashicorp/terraform/terraform" "github.com/mitchellh/cli" @@ -21,7 +21,7 @@ import ( func testOperationPlan(t *testing.T, configDir string) (*backend.Operation, func()) { t.Helper() - _, configLoader, configCleanup := configload.MustLoadConfigForTests(t, configDir) + _, configLoader, configCleanup := initwd.MustLoadConfigForTests(t, configDir) return &backend.Operation{ ConfigDir: configDir, diff --git a/command/command_test.go b/command/command_test.go index 2fa25743b..d4f263c55 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -7,6 +7,8 @@ import ( "encoding/json" "flag" "fmt" + "github.com/hashicorp/terraform/internal/initwd" + "github.com/hashicorp/terraform/registry" "io" "io/ioutil" "log" @@ -141,7 +143,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config t.Helper() dir := filepath.Join(fixtureDir, name) - // FIXME: We're not dealing with the cleanup function here because // this testModule function is used all over and so we don't want to // change its interface at this late stage. @@ -150,9 +151,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config // Test modules usually do not refer to remote sources, and for local // sources only this ultimately just records all of the module paths // in a JSON file so that we can load them below. - diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{}) - if diags.HasErrors() { - t.Fatal(diags.Error()) + inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil)) + instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{}) + if instDiags.HasErrors() { + t.Fatal(instDiags.Err()) } config, snap, diags := loader.LoadConfigWithSnapshot(dir) diff --git a/command/hook_module_install.go b/command/hook_module_install.go index 50df1c0fb..4afa7072c 100644 --- a/command/hook_module_install.go +++ b/command/hook_module_install.go @@ -4,17 +4,17 @@ import ( "fmt" version "github.com/hashicorp/go-version" - "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/internal/initwd" "github.com/mitchellh/cli" ) type uiModuleInstallHooks struct { - configload.InstallHooksImpl + initwd.ModuleInstallHooksImpl Ui cli.Ui ShowLocalPaths bool } -var _ configload.InstallHooks = uiModuleInstallHooks{} +var _ initwd.ModuleInstallHooks = uiModuleInstallHooks{} func (h uiModuleInstallHooks) Download(modulePath, packageAddr string, v *version.Version) { if v != nil { diff --git a/command/meta_config.go b/command/meta_config.go index 7c17c923d..e9bee011e 100644 --- a/command/meta_config.go +++ b/command/meta_config.go @@ -6,12 +6,13 @@ import ( "path/filepath" "sort" - "github.com/hashicorp/hcl2/hcl/hclsyntax" - "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/hcl2/hcl/hclsyntax" "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/configs/configschema" + "github.com/hashicorp/terraform/internal/initwd" + "github.com/hashicorp/terraform/registry" "github.com/hashicorp/terraform/terraform" "github.com/hashicorp/terraform/tfdiags" "github.com/zclconf/go-cty/cty" @@ -165,7 +166,7 @@ func (m *Meta) loadHCLFile(filename string) (hcl.Body, tfdiags.Diagnostics) { // can then be relayed to the end-user. The moduleUiInstallHooks type in // this package has a reasonable implementation for displaying notifications // via a provided cli.Ui. -func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.InstallHooks) tfdiags.Diagnostics { +func (m *Meta) installModules(rootDir string, upgrade bool, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics { var diags tfdiags.Diagnostics rootDir = m.normalizePath(rootDir) @@ -175,14 +176,9 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins return diags } - loader, err := m.initConfigLoader() - if err != nil { - diags = diags.Append(err) - return diags - } - - hclDiags := loader.InstallModules(rootDir, upgrade, hooks) - diags = diags.Append(hclDiags) + inst := m.moduleInstaller() + moreDiags := inst.InstallModules(rootDir, upgrade, hooks) + diags = diags.Append(moreDiags) return diags } @@ -195,18 +191,11 @@ func (m *Meta) installModules(rootDir string, upgrade bool, hooks configload.Ins // can then be relayed to the end-user. The moduleUiInstallHooks type in // this package has a reasonable implementation for displaying notifications // via a provided cli.Ui. -func (m *Meta) initDirFromModule(targetDir string, addr string, hooks configload.InstallHooks) tfdiags.Diagnostics { +func (m *Meta) initDirFromModule(targetDir string, addr string, hooks initwd.ModuleInstallHooks) tfdiags.Diagnostics { var diags tfdiags.Diagnostics targetDir = m.normalizePath(targetDir) - - loader, err := m.initConfigLoader() - if err != nil { - diags = diags.Append(err) - return diags - } - - hclDiags := loader.InitDirFromModule(targetDir, addr, hooks) - diags = diags.Append(hclDiags) + moreDiags := initwd.DirFromModule(targetDir, m.modulesDir(), addr, m.registryClient(), hooks) + diags = diags.Append(moreDiags) return diags } @@ -327,6 +316,18 @@ func (m *Meta) initConfigLoader() (*configload.Loader, error) { return m.configLoader, nil } +// moduleInstaller instantiates and returns a module installer for use by +// "terraform init" (directly or indirectly). +func (m *Meta) moduleInstaller() *initwd.ModuleInstaller { + reg := m.registryClient() + return initwd.NewModuleInstaller(m.modulesDir(), reg) +} + +// registryClient instantiates and returns a new Terraform Registry client. +func (m *Meta) registryClient() *registry.Client { + return registry.NewClient(m.Services, nil) +} + // configValueFromCLI parses a configuration value that was provided in a // context in the CLI where only strings can be provided, such as on the // command line or in an environment variable, and returns the resulting diff --git a/configs/configload/loader.go b/configs/configload/loader.go index a95f5ca68..cabf19f65 100644 --- a/configs/configload/loader.go +++ b/configs/configload/loader.go @@ -71,6 +71,12 @@ func NewLoader(config *Config) (*Loader, error) { return ret, nil } +// ModulesDir returns the path to the directory where the loader will look for +// the local cache of remote module packages. +func (l *Loader) ModulesDir() string { + return l.modules.Dir +} + // Parser returns the underlying parser for this loader. // // This is useful for loading other sorts of files than the module directories diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go deleted file mode 100644 index c6c952dc5..000000000 --- a/configs/configload/loader_install.go +++ /dev/null @@ -1,523 +0,0 @@ -package configload - -import ( - "fmt" - "log" - "os" - "path/filepath" - "strings" - - version "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl2/hcl" - "github.com/hashicorp/terraform/configs" - "github.com/hashicorp/terraform/registry" - "github.com/hashicorp/terraform/registry/regsrc" -) - -// InstallModules analyses the root module in the given directory and installs -// all of its direct and transitive dependencies into the loader's modules -// directory, which must already exist. -// -// Since InstallModules makes possibly-time-consuming calls to remote services, -// a hook interface is supported to allow the caller to be notified when -// each module is installed and, for remote modules, when downloading begins. -// LoadConfig guarantees that two hook calls will not happen concurrently but -// it does not guarantee any particular ordering of hook calls. This mechanism -// is for UI feedback only and does not give the caller any control over the -// process. -// -// If modules are already installed in the target directory, they will be -// skipped unless their source address or version have changed or unless -// the upgrade flag is set. -// -// InstallModules never deletes any directory, except in the case where it -// needs to replace a directory that is already present with a newly-extracted -// package. -// -// If the returned diagnostics contains errors then the module installation -// may have wholly or partially completed. Modules must be loaded in order -// to find their dependencies, so this function does many of the same checks -// as LoadConfig as a side-effect. -// -// This function will panic if called on a loader that cannot install modules. -// Use CanInstallModules to determine if a loader can install modules, or -// refer to the documentation for that method for situations where module -// installation capability is guaranteed. -func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics { - if !l.CanInstallModules() { - panic(fmt.Errorf("InstallModules called on loader that cannot install modules")) - } - - rootMod, diags := l.parser.LoadConfigDir(rootDir) - if rootMod == nil { - return diags - } - - getter := reusingGetter{} - instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks, getter) - diags = append(diags, instDiags...) - - return diags -} - -func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks, getter reusingGetter) hcl.Diagnostics { - var diags hcl.Diagnostics - - if hooks == nil { - // Use our no-op implementation as a placeholder - hooks = InstallHooksImpl{} - } - - // Create a manifest record for the root module. This will be used if - // there are any relative-pathed modules in the root. - l.modules.manifest[""] = moduleRecord{ - Key: "", - Dir: rootDir, - } - - _, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc( - func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) { - - key := manifestKey(req.Path) - instPath := l.packageInstallPath(req.Path) - - log.Printf("[DEBUG] Module installer: begin %s", key) - - // First we'll check if we need to upgrade/replace an existing - // installed module, and delete it out of the way if so. - replace := upgrade - if !replace { - record, recorded := l.modules.manifest[key] - switch { - case !recorded: - log.Printf("[TRACE] %s is not yet installed", key) - replace = true - case record.SourceAddr != req.SourceAddr: - log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr) - replace = true - case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version): - log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required) - replace = true - } - } - - // If we _are_ planning to replace this module, then we'll remove - // it now so our installation code below won't conflict with any - // existing remnants. - if replace { - if _, recorded := l.modules.manifest[key]; recorded { - log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key) - } - delete(l.modules.manifest, key) - // Deleting a module invalidates all of its descendent modules too. - keyPrefix := key + "." - for subKey := range l.modules.manifest { - if strings.HasPrefix(subKey, keyPrefix) { - if _, recorded := l.modules.manifest[subKey]; recorded { - log.Printf("[TRACE] also discarding downstream %s", subKey) - } - delete(l.modules.manifest, subKey) - } - } - } - - record, recorded := l.modules.manifest[key] - if !recorded { - // Clean up any stale cache directory that might be present. - // If this is a local (relative) source then the dir will - // not exist, but we'll ignore that. - log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key) - err := l.modules.FS.RemoveAll(instPath) - if err != nil && !os.IsNotExist(err) { - log.Printf("[TRACE] failed to remove %s: %s", key, err) - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to remove local module cache", - Detail: fmt.Sprintf( - "Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s", - instPath, err, - ), - Subject: &req.CallRange, - }) - return nil, nil, diags - } - } else { - // If this module is already recorded and its root directory - // exists then we will just load what's already there and - // keep our existing record. - info, err := l.modules.FS.Stat(record.Dir) - if err == nil && info.IsDir() { - mod, mDiags := l.parser.LoadConfigDir(record.Dir) - diags = append(diags, mDiags...) - - log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir) - return mod, record.Version, diags - } - } - - // If we get down here then it's finally time to actually install - // the module. There are some variants to this process depending - // on what type of module source address we have. - switch { - - case isLocalSourceAddr(req.SourceAddr): - log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr) - mod, mDiags := l.installLocalModule(req, key, hooks) - diags = append(diags, mDiags...) - return mod, nil, diags - - case isRegistrySourceAddr(req.SourceAddr): - addr, err := regsrc.ParseModuleSource(req.SourceAddr) - if err != nil { - // Should never happen because isRegistrySourceAddr already validated - panic(err) - } - log.Printf("[TRACE] %s is a registry module at %s", key, addr) - - mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks, getter) - diags = append(diags, mDiags...) - return mod, v, diags - - default: - log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr) - - mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks, getter) - diags = append(diags, mDiags...) - return mod, nil, diags - } - - }, - )) - diags = append(diags, cDiags...) - - err := l.modules.writeModuleManifestSnapshot() - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to update module manifest", - Detail: fmt.Sprintf("Unable to write the module manifest file: %s", err), - }) - } - - return diags -} - -// CanInstallModules returns true if InstallModules can be used with this -// loader. -// -// Loaders created with NewLoader can always install modules. Loaders created -// from plan files (where the configuration is embedded in the plan file itself) -// cannot install modules, because the plan file is read-only. -func (l *Loader) CanInstallModules() bool { - return l.modules.CanInstall -} - -func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) { - var diags hcl.Diagnostics - - parentKey := manifestKey(req.Parent.Path) - parentRecord, recorded := l.modules.manifest[parentKey] - if !recorded { - // This is indicative of a bug rather than a user-actionable error - panic(fmt.Errorf("missing manifest record for parent module %s", parentKey)) - } - - if len(req.VersionConstraint.Required) != 0 { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid version constraint", - Detail: "A version constraint cannot be applied to a module at a relative local path.", - Subject: &req.VersionConstraint.DeclRange, - }) - } - - // For local sources we don't actually need to modify the - // filesystem at all because the parent already wrote - // the files we need, and so we just load up what's already here. - newDir := filepath.Join(parentRecord.Dir, req.SourceAddr) - log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir) - mod, mDiags := l.parser.LoadConfigDir(newDir) - if mod == nil { - // nil indicates missing or unreadable directory, so we'll - // discard the returned diags and return a more specific - // error message here. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Unreadable module directory", - Detail: fmt.Sprintf("The directory %s could not be read.", newDir), - Subject: &req.SourceAddrRange, - }) - } else { - diags = append(diags, mDiags...) - } - - // Note the local location in our manifest. - l.modules.manifest[key] = moduleRecord{ - Key: key, - Dir: newDir, - SourceAddr: req.SourceAddr, - } - log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir) - hooks.Install(key, nil, newDir) - - return mod, diags -} - -func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks, getter reusingGetter) (*configs.Module, *version.Version, hcl.Diagnostics) { - var diags hcl.Diagnostics - - hostname, err := addr.SvcHost() - if err != nil { - // If it looks like the user was trying to use punycode then we'll generate - // a specialized error for that case. We require the unicode form of - // hostname so that hostnames are always human-readable in configuration - // and punycode can't be used to hide a malicious module hostname. - if strings.HasPrefix(addr.RawHost.Raw, "xn--") { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid module registry hostname", - Detail: "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.", - Subject: &req.SourceAddrRange, - }) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid module registry hostname", - Detail: "The hostname portion of this source address is not a valid hostname.", - Subject: &req.SourceAddrRange, - }) - } - return nil, nil, diags - } - - reg := l.modules.Registry - - log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname) - resp, err := reg.ModuleVersions(addr) - if err != nil { - if registry.IsModuleNotFound(err) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Module not found", - Detail: fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname), - Subject: &req.SourceAddrRange, - }) - } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Error accessing remote module registry", - Detail: fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err), - Subject: &req.SourceAddrRange, - }) - } - return nil, nil, diags - } - - // The response might contain information about dependencies to allow us - // to potentially optimize future requests, but we don't currently do that - // and so for now we'll just take the first item which is guaranteed to - // be the address we requested. - if len(resp.Modules) < 1 { - // Should never happen, but since this is a remote service that may - // be implemented by third-parties we will handle it gracefully. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid response from remote module registry", - Detail: fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname), - Subject: &req.SourceAddrRange, - }) - return nil, nil, diags - } - - modMeta := resp.Modules[0] - - var latestMatch *version.Version - var latestVersion *version.Version - for _, mv := range modMeta.Versions { - v, err := version.NewVersion(mv.Version) - if err != nil { - // Should never happen if the registry server is compliant with - // the protocol, but we'll warn if not to assist someone who - // might be developing a module registry server. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "Invalid response from remote module registry", - Detail: fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version), - Subject: &req.SourceAddrRange, - }) - continue - } - - // If we've found a pre-release version then we'll ignore it unless - // it was exactly requested. - if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() { - log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v) - continue - } - - if latestVersion == nil || v.GreaterThan(latestVersion) { - latestVersion = v - } - - if req.VersionConstraint.Required.Check(v) { - if latestMatch == nil || v.GreaterThan(latestMatch) { - latestMatch = v - } - } - } - - if latestVersion == nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Module has no versions", - Detail: fmt.Sprintf("The specified module does not have any available versions."), - Subject: &req.SourceAddrRange, - }) - return nil, nil, diags - } - - if latestMatch == nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Unresolvable module version constraint", - Detail: fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion), - Subject: &req.VersionConstraint.DeclRange, - }) - return nil, nil, diags - } - - // Report up to the caller that we're about to start downloading. - packageAddr, _ := splitAddrSubdir(req.SourceAddr) - hooks.Download(key, packageAddr, latestMatch) - - // If we manage to get down here then we've found a suitable version to - // install, so we need to ask the registry where we should download it from. - // The response to this is a go-getter-style address string. - dlAddr, err := reg.ModuleLocation(addr, latestMatch.String()) - if err != nil { - log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err) - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid response from remote module registry", - Detail: fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch), - Subject: &req.VersionConstraint.DeclRange, - }) - return nil, nil, diags - } - - log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr) - - modDir, err := getter.getWithGoGetter(instPath, dlAddr) - if err != nil { - // Errors returned by go-getter have very inconsistent quality as - // end-user error messages, but for now we're accepting that because - // we have no way to recognize any specific errors to improve them - // and masking the error entirely would hide valuable diagnostic - // information from the user. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to download module", - Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err), - Subject: &req.CallRange, - }) - return nil, nil, diags - } - - log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir) - - if addr.RawSubmodule != "" { - // Append the user's requested subdirectory to any subdirectory that - // was implied by any of the nested layers we expanded within go-getter. - modDir = filepath.Join(modDir, addr.RawSubmodule) - } - - log.Printf("[TRACE] %s should now be at %s", key, modDir) - - // Finally we are ready to try actually loading the module. - mod, mDiags := l.parser.LoadConfigDir(modDir) - if mod == nil { - // nil indicates missing or unreadable directory, so we'll - // discard the returned diags and return a more specific - // error message here. For registry modules this actually - // indicates a bug in the code above, since it's not the - // user's responsibility to create the directory in this case. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Unreadable module directory", - Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir), - Subject: &req.CallRange, - }) - } else { - diags = append(diags, mDiags...) - } - - // Note the local location in our manifest. - l.modules.manifest[key] = moduleRecord{ - Key: key, - Version: latestMatch, - Dir: modDir, - SourceAddr: req.SourceAddr, - } - log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir) - hooks.Install(key, latestMatch, modDir) - - return mod, latestMatch, diags -} - -func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks, getter reusingGetter) (*configs.Module, hcl.Diagnostics) { - var diags hcl.Diagnostics - - // Report up to the caller that we're about to start downloading. - packageAddr, _ := splitAddrSubdir(req.SourceAddr) - hooks.Download(key, packageAddr, nil) - - modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr) - if err != nil { - // Errors returned by go-getter have very inconsistent quality as - // end-user error messages, but for now we're accepting that because - // we have no way to recognize any specific errors to improve them - // and masking the error entirely would hide valuable diagnostic - // information from the user. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to download module", - Detail: fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err), - Subject: &req.SourceAddrRange, - }) - return nil, diags - } - - log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir) - - mod, mDiags := l.parser.LoadConfigDir(modDir) - if mod == nil { - // nil indicates missing or unreadable directory, so we'll - // discard the returned diags and return a more specific - // error message here. For registry modules this actually - // indicates a bug in the code above, since it's not the - // user's responsibility to create the directory in this case. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Unreadable module directory", - Detail: fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir), - Subject: &req.CallRange, - }) - } else { - diags = append(diags, mDiags...) - } - - // Note the local location in our manifest. - l.modules.manifest[key] = moduleRecord{ - Key: key, - Dir: modDir, - SourceAddr: req.SourceAddr, - } - log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir) - hooks.Install(key, nil, modDir) - - return mod, diags -} - -func (l *Loader) packageInstallPath(modulePath []string) string { - return filepath.Join(l.modules.Dir, strings.Join(modulePath, ".")) -} diff --git a/configs/configload/loader_install_hooks.go b/configs/configload/loader_install_hooks.go deleted file mode 100644 index 058369414..000000000 --- a/configs/configload/loader_install_hooks.go +++ /dev/null @@ -1,34 +0,0 @@ -package configload - -import version "github.com/hashicorp/go-version" - -// InstallHooks is an interface used to provide notifications about the -// installation process being orchestrated by InstallModules. -// -// This interface may have new methods added in future, so implementers should -// embed InstallHooksImpl to get no-op implementations of any unimplemented -// methods. -type InstallHooks interface { - // Download is called for modules that are retrieved from a remote source - // before that download begins, to allow a caller to give feedback - // on progress through a possibly-long sequence of downloads. - Download(moduleAddr, packageAddr string, version *version.Version) - - // Install is called for each module that is installed, even if it did - // not need to be downloaded from a remote source. - Install(moduleAddr string, version *version.Version, localPath string) -} - -// InstallHooksImpl is a do-nothing implementation of InstallHooks that -// can be embedded in another implementation struct to allow only partial -// implementation of the interface. -type InstallHooksImpl struct { -} - -func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) { -} - -func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) { -} - -var _ InstallHooks = InstallHooksImpl{} diff --git a/configs/configload/loader_load.go b/configs/configload/loader_load.go index 104a31d20..92d09b12c 100644 --- a/configs/configload/loader_load.go +++ b/configs/configload/loader_load.go @@ -24,6 +24,18 @@ func (l *Loader) LoadConfig(rootDir string) (*configs.Config, hcl.Diagnostics) { return nil, diags } + // Refresh the manifest snapshot in case anything new has been installed + // since we last refreshed it. + err := l.modules.readModuleManifestSnapshot() + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to read module manifest", + Detail: fmt.Sprintf("Terraform failed to read its manifest of locally-cached modules: %s.", err), + }) + return nil, diags + } + cfg, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(l.moduleWalkerLoad)) diags = append(diags, cDiags...) @@ -39,7 +51,7 @@ func (l *Loader) moduleWalkerLoad(req *configs.ModuleRequest) (*configs.Module, // do verify that the manifest and the configuration are in agreement // so that we can prompt the user to run "terraform init" if not. - key := manifestKey(req.Path) + key := l.modules.manifest.ModuleKey(req.Path) record, exists := l.modules.manifest[key] if !exists { diff --git a/configs/configload/loader_snapshot.go b/configs/configload/loader_snapshot.go index 13193e05a..44c643973 100644 --- a/configs/configload/loader_snapshot.go +++ b/configs/configload/loader_snapshot.go @@ -10,10 +10,9 @@ import ( version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" - "github.com/spf13/afero" - - "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/internal/modsdir" + "github.com/spf13/afero" ) // LoadConfigWithSnapshot is a variant of LoadConfig that also simultaneously @@ -83,7 +82,7 @@ type Snapshot struct { func NewEmptySnapshot() *Snapshot { return &Snapshot{ Modules: map[string]*SnapshotModule{ - manifestKey(addrs.RootModule): &SnapshotModule{ + "": &SnapshotModule{ Files: map[string][]byte{}, }, }, @@ -111,11 +110,11 @@ type SnapshotModule struct { // moduleManifest constructs a module manifest based on the contents of // the receiving snapshot. -func (s *Snapshot) moduleManifest() moduleManifest { - ret := make(moduleManifest) +func (s *Snapshot) moduleManifest() modsdir.Manifest { + ret := make(modsdir.Manifest) for k, modSnap := range s.Modules { - ret[k] = moduleRecord{ + ret[k] = modsdir.Record{ Key: k, Dir: modSnap.Dir, SourceAddr: modSnap.SourceAddr, @@ -137,7 +136,7 @@ func (l *Loader) makeModuleWalkerSnapshot(snap *Snapshot) configs.ModuleWalker { return mod, v, diags } - key := manifestKey(req.Path) + key := l.modules.manifest.ModuleKey(req.Path) record, exists := l.modules.manifest[key] if !exists { diff --git a/configs/configload/testing.go b/configs/configload/testing.go index a85d4a294..86ca9d10b 100644 --- a/configs/configload/testing.go +++ b/configs/configload/testing.go @@ -4,9 +4,6 @@ import ( "io/ioutil" "os" "testing" - - "github.com/hashicorp/terraform/configs" - "github.com/hashicorp/terraform/tfdiags" ) // NewLoaderForTests is a variant of NewLoader that is intended to be more @@ -44,58 +41,3 @@ func NewLoaderForTests(t *testing.T) (*Loader, func()) { return loader, cleanup } - -// LoadConfigForTests is a convenience wrapper around NewLoaderForTests, -// Loader.InstallModules and Loader.LoadConfig that allows a test configuration -// to be loaded in a single step. -// -// If module installation fails, t.Fatal (or similar) is called to halt -// execution of the test, under the assumption that installation failures are -// not expected. If installation failures _are_ expected then use -// NewLoaderForTests and work with the loader object directly. If module -// installation succeeds but generates warnings, these warnings are discarded. -// -// If installation succeeds but errors are detected during loading then a -// possibly-incomplete config is returned along with error diagnostics. The -// test run is not aborted in this case, so that the caller can make assertions -// against the returned diagnostics. -// -// As with NewLoaderForTests, a cleanup function is returned which must be -// called before the test completes in order to remove the temporary -// modules directory. -func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func(), tfdiags.Diagnostics) { - t.Helper() - - var diags tfdiags.Diagnostics - - loader, cleanup := NewLoaderForTests(t) - hclDiags := loader.InstallModules(rootDir, true, InstallHooksImpl{}) - if diags.HasErrors() { - cleanup() - diags = diags.Append(hclDiags) - t.Fatal(diags.Err()) - return nil, nil, cleanup, diags - } - - config, hclDiags := loader.LoadConfig(rootDir) - diags = diags.Append(hclDiags) - return config, loader, cleanup, diags -} - -// MustLoadConfigForTests is a variant of LoadConfigForTests which calls -// t.Fatal (or similar) if there are any errors during loading, and thus -// does not return diagnostics at all. -// -// This is useful for concisely writing tests that don't expect errors at -// all. For tests that expect errors and need to assert against them, use -// LoadConfigForTests instead. -func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *Loader, func()) { - t.Helper() - - config, loader, cleanup, diags := LoadConfigForTests(t, rootDir) - if diags.HasErrors() { - cleanup() - t.Fatal(diags.Err()) - } - return config, loader, cleanup -} diff --git a/helper/resource/testing.go b/helper/resource/testing.go index fe3df8ddf..be0ea13bd 100644 --- a/helper/resource/testing.go +++ b/helper/resource/testing.go @@ -26,6 +26,7 @@ import ( "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/helper/logging" + "github.com/hashicorp/terraform/internal/initwd" "github.com/hashicorp/terraform/providers" "github.com/hashicorp/terraform/states" "github.com/hashicorp/terraform/terraform" @@ -832,6 +833,12 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err return nil, fmt.Errorf("Error creating child modules directory: %s", err) } + inst := initwd.NewModuleInstaller(modulesDir, nil) + installDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{}) + if installDiags.HasErrors() { + return nil, installDiags.Err() + } + loader, err := configload.NewLoader(&configload.Config{ ModulesDir: modulesDir, }) @@ -839,11 +846,6 @@ func testConfig(opts terraform.ContextOpts, step TestStep) (*configs.Config, err return nil, fmt.Errorf("failed to create config loader: %s", err) } - installDiags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{}) - if installDiags.HasErrors() { - return nil, installDiags - } - config, configDiags := loader.LoadConfig(cfgPath) if configDiags.HasErrors() { return nil, configDiags diff --git a/internal/earlyconfig/config_build.go b/internal/earlyconfig/config_build.go index f07c5982f..770d5dfbe 100644 --- a/internal/earlyconfig/config_build.go +++ b/internal/earlyconfig/config_build.go @@ -3,6 +3,7 @@ package earlyconfig import ( "fmt" "sort" + "strings" version "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-config-inspect/tfconfig" @@ -41,14 +42,18 @@ func buildChildModules(parent *Config, walker ModuleWalker) (map[string]*Config, copy(path, parent.Path) path[len(path)-1] = call.Name - vc, err := version.NewConstraint(call.Version) - if err != nil { - diags = diags.Append(wrapDiagnostic(tfconfig.Diagnostic{ - Severity: tfconfig.DiagError, - Summary: "Invalid version constraint", - Detail: fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint: %s.", callName, call.Pos.Filename, call.Pos.Line, err), - })) - continue + var vc version.Constraints + if strings.TrimSpace(call.Version) != "" { + var err error + vc, err = version.NewConstraint(call.Version) + if err != nil { + diags = diags.Append(wrapDiagnostic(tfconfig.Diagnostic{ + Severity: tfconfig.DiagError, + Summary: "Invalid version constraint", + Detail: fmt.Sprintf("Module %q (declared at %s line %d) has invalid version constraint %q: %s.", callName, call.Pos.Filename, call.Pos.Line, call.Version, err), + })) + continue + } } req := ModuleRequest{ diff --git a/internal/init/copy_dir.go b/internal/initwd/copy_dir.go similarity index 99% rename from internal/init/copy_dir.go rename to internal/initwd/copy_dir.go index 36a4d2c1c..8e414cdaa 100644 --- a/internal/init/copy_dir.go +++ b/internal/initwd/copy_dir.go @@ -1,4 +1,4 @@ -package init +package initwd import ( "io" diff --git a/internal/init/doc.go b/internal/initwd/doc.go similarity index 67% rename from internal/init/doc.go rename to internal/initwd/doc.go index 7710e6ac1..02e372377 100644 --- a/internal/init/doc.go +++ b/internal/initwd/doc.go @@ -1,7 +1,7 @@ -// Package init contains various helper functions used by the "terraform init" +// Package initwd contains various helper functions used by the "terraform init" // command. // // These functions may also be used from testing code to simulate the behaviors // of "terraform init" against test fixtures, but should not be used elsewhere // in the main code. -package init +package initwd diff --git a/configs/configload/loader_init_from_module.go b/internal/initwd/from_module.go similarity index 54% rename from configs/configload/loader_init_from_module.go rename to internal/initwd/from_module.go index c383e2f4a..63a6dd0af 100644 --- a/configs/configload/loader_init_from_module.go +++ b/internal/initwd/from_module.go @@ -1,7 +1,9 @@ -package configload +package initwd import ( "fmt" + "github.com/hashicorp/terraform/internal/earlyconfig" + "io/ioutil" "log" "os" "path/filepath" @@ -9,14 +11,16 @@ import ( "strings" version "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl2/hcl" - "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/hashicorp/terraform/internal/modsdir" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/tfdiags" ) const initFromModuleRootCallName = "root" const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "." -// InitDirFromModule populates the given directory (which must exist and be +// DirFromModule populates the given directory (which must exist and be // empty) with the contents of the module at the given source address. // // It does this by installing the given module and all of its descendent @@ -34,13 +38,8 @@ const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "." // references using ../ from that module to be unresolvable. Error diagnostics // are produced in that case, to prompt the user to rewrite the source strings // to be absolute references to the original remote module. -// -// This can be installed only on a loder that can install modules, and will -// panic otherwise. Use CanInstallModules to determine if this method can be -// used, or refer to the documentation of that method for situations where -// install ability is guaranteed. -func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHooks) hcl.Diagnostics { - var diags hcl.Diagnostics +func DirFromModule(rootDir, modulesDir, sourceAddr string, reg *registry.Client, hooks ModuleInstallHooks) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics // The way this function works is pretty ugly, but we accept it because // -from-module is a less important case than normal module installation @@ -49,20 +48,20 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook // The target directory must exist but be empty. { - entries, err := l.modules.FS.ReadDir(rootDir) + entries, err := ioutil.ReadDir(rootDir) if err != nil { if os.IsNotExist(err) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Target directory does not exist", - Detail: fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Target directory does not exist", + fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir), + )) } else { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to read target directory", - Detail: fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to read target directory", + fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err), + )) } return diags } @@ -74,57 +73,36 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook haveEntries = true } if haveEntries { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Can't populate non-empty directory", - Detail: fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Can't populate non-empty directory", + fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir), + )) return diags } } - // We use a hidden sub-loader to manage our inner installation directory, - // but it shares dependencies with the receiver to allow it to access the - // same remote resources and ensure it populates the same source code - // cache in case . - subLoader := &Loader{ - parser: l.parser, - modules: l.modules, // this is a shallow copy, so we can safely mutate below - } - - // Our sub-loader will have its own independent manifest and install - // directory, so we can install with it and know we won't interfere - // with the receiver. - subLoader.modules.manifest = make(moduleManifest) - subLoader.modules.Dir = filepath.Join(rootDir, ".terraform/init-from-module") - - log.Printf("[DEBUG] using a child module loader in %s to initialize working directory from %q", subLoader.modules.Dir, sourceAddr) - - subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) // if this fails then we'll fail on MkdirAll below too - - err := subLoader.modules.FS.MkdirAll(subLoader.modules.Dir, os.ModePerm) + instDir := filepath.Join(rootDir, ".terraform/init-from-module") + inst := NewModuleInstaller(instDir, reg) + log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddr) + os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too + err := os.MkdirAll(instDir, os.ModePerm) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to create temporary directory", - Detail: fmt.Sprintf("Failed to create temporary directory %s: %s.", subLoader.modules.Dir, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to create temporary directory", + fmt.Sprintf("Failed to create temporary directory %s: %s.", instDir, err), + )) return diags } + instManifest := make(modsdir.Manifest) + retManifest := make(modsdir.Manifest) + fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr) - fakeRange := hcl.Range{ + fakePos := tfconfig.SourcePos{ Filename: fakeFilename, - Start: hcl.Pos{ - Line: 1, - Column: 1, - Byte: 0, - }, - End: hcl.Pos{ - Line: 1, - Column: len(fakeFilename) + 1, // not accurate if the address contains unicode, but irrelevant since we have no source cache for this anyway - Byte: len(fakeFilename), - }, + Line: 1, } // -from-module allows relative paths but it's different than a normal @@ -146,16 +124,12 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook // Now we need to create an artificial root module that will seed our // installation process. - fakeRootModule := &configs.Module{ - ModuleCalls: map[string]*configs.ModuleCall{ - initFromModuleRootCallName: &configs.ModuleCall{ - Name: initFromModuleRootCallName, - - SourceAddr: sourceAddr, - SourceAddrRange: fakeRange, - SourceSet: true, - - DeclRange: fakeRange, + fakeRootModule := &tfconfig.Module{ + ModuleCalls: map[string]*tfconfig.ModuleCall{ + initFromModuleRootCallName: { + Name: initFromModuleRootCallName, + Source: sourceAddr, + Pos: fakePos, }, }, } @@ -167,7 +141,7 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook Wrapped: hooks, } getter := reusingGetter{} - instDiags := subLoader.installDescendentModules(fakeRootModule, rootDir, true, wrapHooks, getter) + instDiags := inst.installDescendentModules(fakeRootModule, rootDir, instManifest, true, wrapHooks, getter) diags = append(diags, instDiags...) if instDiags.HasErrors() { return diags @@ -175,26 +149,24 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook // If all of that succeeded then we'll now migrate what was installed // into the final directory structure. - modulesDir := l.modules.Dir - err = subLoader.modules.FS.MkdirAll(modulesDir, os.ModePerm) + err = os.MkdirAll(modulesDir, os.ModePerm) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to create local modules directory", - Detail: fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to create local modules directory", + fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err), + )) return diags } - manifest := subLoader.modules.manifest - recordKeys := make([]string, 0, len(manifest)) - for k := range manifest { + recordKeys := make([]string, 0, len(instManifest)) + for k := range instManifest { recordKeys = append(recordKeys, k) } sort.Strings(recordKeys) for _, recordKey := range recordKeys { - record := manifest[recordKey] + record := instManifest[recordKey] if record.Key == initFromModuleRootCallName { // We've found the module the user requested, which we must @@ -202,11 +174,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir) err := copyDir(rootDir, record.Dir) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to copy root module", - Detail: fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to copy root module", + fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err), + )) continue } @@ -215,36 +187,37 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook // and must thus be rewritten to be absolute addresses again. // For now we can't do this rewriting automatically, but we'll // generate an error to help the user do it manually. - mod, _ := l.parser.LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway - for _, mc := range mod.ModuleCalls { - if pathTraversesUp(sourceAddr) { - packageAddr, givenSubdir := splitAddrSubdir(sourceAddr) - newSubdir := filepath.Join(givenSubdir, mc.SourceAddr) - if pathTraversesUp(newSubdir) { - // This should never happen in any reasonable - // configuration since this suggests a path that - // traverses up out of the package root. We'll just - // ignore this, since we'll fail soon enough anyway - // trying to resolve this path when this module is - // loaded. + mod, _ := earlyconfig.LoadModule(rootDir) // ignore diagnostics since we're just doing value-add here anyway + if mod != nil { + for _, mc := range mod.ModuleCalls { + if pathTraversesUp(mc.Source) { + packageAddr, givenSubdir := splitAddrSubdir(sourceAddr) + newSubdir := filepath.Join(givenSubdir, mc.Source) + if pathTraversesUp(newSubdir) { + // This should never happen in any reasonable + // configuration since this suggests a path that + // traverses up out of the package root. We'll just + // ignore this, since we'll fail soon enough anyway + // trying to resolve this path when this module is + // loaded. + continue + } + + var newAddr = packageAddr + if newSubdir != "" { + newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir)) + } + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Root module references parent directory", + fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr), + )) continue } - - var newAddr = packageAddr - if newSubdir != "" { - newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir)) - } - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Root module references parent directory", - Detail: fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr), - Subject: &mc.SourceAddrRange, - }) - continue } } - l.modules.manifest[""] = moduleRecord{ + retManifest[""] = modsdir.Record{ Key: "", Dir: rootDir, } @@ -259,8 +232,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook } newKey := record.Key[len(initFromModuleRootKeyPrefix):] - instPath := filepath.Join(l.modules.Dir, newKey) - tempPath := filepath.Join(subLoader.modules.Dir, record.Key) + instPath := filepath.Join(modulesDir, newKey) + tempPath := filepath.Join(instDir, record.Key) // tempPath won't be present for a module that was installed from // a relative path, so in that case we just record the installation @@ -268,11 +241,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook // of its parent. if _, err := os.Stat(tempPath); err != nil { if !os.IsNotExist(err) { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to stat temporary module install directory", - Detail: fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to stat temporary module install directory", + fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err), + )) continue } @@ -283,8 +256,8 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook parentKey = "" // parent is the root module } - parentOld := manifest[initFromModuleRootKeyPrefix+parentKey] - parentNew := l.modules.manifest[parentKey] + parentOld := instManifest[initFromModuleRootKeyPrefix+parentKey] + parentNew := retManifest[parentKey] // We need to figure out which portion of our directory is the // parent package path and which portion is the subdirectory @@ -301,18 +274,18 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook newRecord := record // shallow copy newRecord.Dir = newDir newRecord.Key = newKey - l.modules.manifest[newKey] = newRecord + retManifest[newKey] = newRecord hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) continue } - err = subLoader.modules.FS.MkdirAll(instPath, os.ModePerm) + err = os.MkdirAll(instPath, os.ModePerm) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to create module install directory", - Detail: fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to create module install directory", + fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err), + )) continue } @@ -321,11 +294,11 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath) err := copyDir(instPath, tempPath) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to copy descendent module", - Detail: fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to copy descendent module", + fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err), + )) continue } @@ -339,23 +312,23 @@ func (l *Loader) InitDirFromModule(rootDir, sourceAddr string, hooks InstallHook newRecord := record // shallow copy newRecord.Dir = filepath.Join(instPath, subDir) newRecord.Key = newKey - l.modules.manifest[newKey] = newRecord + retManifest[newKey] = newRecord hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) } - err = l.modules.writeModuleManifestSnapshot() + retManifest.WriteSnapshotToDir(modulesDir) if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Failed to write module manifest", - Detail: fmt.Sprintf("Error writing module manifest: %s.", err), - }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Failed to write module manifest", + fmt.Sprintf("Error writing module manifest: %s.", err), + )) } if !diags.HasErrors() { // Try to clean up our temporary directory, but don't worry if we don't // succeed since it shouldn't hurt anything. - subLoader.modules.FS.RemoveAll(subLoader.modules.Dir) + os.RemoveAll(instDir) } return diags @@ -373,8 +346,8 @@ func pathTraversesUp(path string) bool { // does its own installation steps after the initial installation pass // has completed. type installHooksInitDir struct { - Wrapped InstallHooks - InstallHooksImpl + Wrapped ModuleInstallHooks + ModuleInstallHooksImpl } func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) { diff --git a/configs/configload/loader_init_from_module_test.go b/internal/initwd/from_module_test.go similarity index 79% rename from configs/configload/loader_init_from_module_test.go rename to internal/initwd/from_module_test.go index f08540297..5ebebae6b 100644 --- a/configs/configload/loader_init_from_module_test.go +++ b/internal/initwd/from_module_test.go @@ -1,4 +1,4 @@ -package configload +package initwd import ( "os" @@ -8,20 +8,25 @@ import ( version "github.com/hashicorp/go-version" "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/tfdiags" ) -func TestLoaderInitDirFromModule_registry(t *testing.T) { +func TestDirFromModule_registry(t *testing.T) { if os.Getenv("TF_ACC") == "" { t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it") } fixtureDir := filepath.Clean("test-fixtures/empty") - loader, done := tempChdirLoader(t, fixtureDir) + dir, done := tempChdir(t, fixtureDir) + modsDir := filepath.Join(dir, ".terraform/modules") defer done() hooks := &testInstallHooks{} - diags := loader.InitDirFromModule(".", "hashicorp/module-installer-acctest/aws//examples/main", hooks) + reg := registry.NewClient(nil, nil) + diags := DirFromModule(dir, modsDir, "hashicorp/module-installer-acctest/aws//examples/main", reg, hooks) assertNoDiagnostics(t, diags) v := version.Must(version.NewVersion("0.0.1")) @@ -65,10 +70,17 @@ func TestLoaderInitDirFromModule_registry(t *testing.T) { return } + loader, err := configload.NewLoader(&configload.Config{ + ModulesDir: modsDir, + }) + if err != nil { + t.Fatal(err) + } + // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) config, loadDiags := loader.LoadConfig(".") - if assertNoDiagnostics(t, loadDiags) { + if assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) { return } diff --git a/internal/init/getter.go b/internal/initwd/getter.go similarity index 98% rename from internal/init/getter.go rename to internal/initwd/getter.go index 1270f3533..ada192b59 100644 --- a/internal/init/getter.go +++ b/internal/initwd/getter.go @@ -1,4 +1,4 @@ -package init +package initwd import ( "fmt" @@ -23,6 +23,7 @@ var goGetterDetectors = []getter.Detector{ new(getter.GitHubDetector), new(getter.BitBucketDetector), new(getter.S3Detector), + new(getter.FileDetector), } var goGetterNoDetectors = []getter.Detector{} @@ -44,6 +45,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{ } var goGetterGetters = map[string]getter.Getter{ + "file": new(getter.FileGetter), "git": new(getter.GitGetter), "hg": new(getter.HgGetter), "s3": new(getter.S3Getter), diff --git a/internal/init/inode.go b/internal/initwd/inode.go similarity index 96% rename from internal/init/inode.go rename to internal/initwd/inode.go index c9ad6ef40..1150b093c 100644 --- a/internal/init/inode.go +++ b/internal/initwd/inode.go @@ -1,6 +1,6 @@ // +build linux darwin openbsd netbsd solaris dragonfly -package init +package initwd import ( "fmt" diff --git a/internal/init/inode_freebsd.go b/internal/initwd/inode_freebsd.go similarity index 95% rename from internal/init/inode_freebsd.go rename to internal/initwd/inode_freebsd.go index ef3d4c2e5..30532f54a 100644 --- a/internal/init/inode_freebsd.go +++ b/internal/initwd/inode_freebsd.go @@ -1,6 +1,6 @@ // +build freebsd -package init +package initwd import ( "fmt" diff --git a/internal/init/inode_windows.go b/internal/initwd/inode_windows.go similarity index 89% rename from internal/init/inode_windows.go rename to internal/initwd/inode_windows.go index 568ff2480..3ed58e4bf 100644 --- a/internal/init/inode_windows.go +++ b/internal/initwd/inode_windows.go @@ -1,6 +1,6 @@ // +build windows -package init +package initwd // no syscall.Stat_t on windows, return 0 for inodes func inode(path string) (uint64, error) { diff --git a/internal/init/module_install.go b/internal/initwd/module_install.go similarity index 88% rename from internal/init/module_install.go rename to internal/initwd/module_install.go index add726610..b9031aa57 100644 --- a/internal/init/module_install.go +++ b/internal/initwd/module_install.go @@ -1,4 +1,4 @@ -package init +package initwd import ( "fmt" @@ -58,7 +58,9 @@ func NewModuleInstaller(modsDir string, reg *registry.Client) *ModuleInstaller { // Use CanInstallModules to determine if a loader can install modules, or // refer to the documentation for that method for situations where module // installation capability is guaranteed. -func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) tfdiags.Diagnostics { +func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks ModuleInstallHooks) tfdiags.Diagnostics { + log.Printf("[TRACE] ModuleInstaller: installing child modules for %s into %s", rootDir, i.modsDir) + rootMod, diags := earlyconfig.LoadModule(rootDir) if rootMod == nil { return diags @@ -81,12 +83,12 @@ func (i *ModuleInstaller) InstallModules(rootDir string, upgrade bool, hooks Ins return diags } -func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks InstallHooks, getter reusingGetter) tfdiags.Diagnostics { +func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, rootDir string, manifest modsdir.Manifest, upgrade bool, hooks ModuleInstallHooks, getter reusingGetter) tfdiags.Diagnostics { var diags tfdiags.Diagnostics if hooks == nil { // Use our no-op implementation as a placeholder - hooks = InstallHooksImpl{} + hooks = ModuleInstallHooksImpl{} } // Create a manifest record for the root module. This will be used if @@ -111,13 +113,13 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo record, recorded := manifest[key] switch { case !recorded: - log.Printf("[TRACE] %s is not yet installed", key) + log.Printf("[TRACE] ModuleInstaller: %s is not yet installed", key) replace = true case record.SourceAddr != req.SourceAddr: - log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr) + log.Printf("[TRACE] ModuleInstaller: %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr) replace = true case record.Version != nil && !req.VersionConstraints.Check(record.Version): - log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints) + log.Printf("[TRACE] ModuleInstaller: %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraints) replace = true } } @@ -127,7 +129,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo // existing remnants. if replace { if _, recorded := manifest[key]; recorded { - log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key) + log.Printf("[TRACE] ModuleInstaller: discarding previous record of %s prior to reinstall", key) } delete(manifest, key) // Deleting a module invalidates all of its descendent modules too. @@ -135,7 +137,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo for subKey := range manifest { if strings.HasPrefix(subKey, keyPrefix) { if _, recorded := manifest[subKey]; recorded { - log.Printf("[TRACE] also discarding downstream %s", subKey) + log.Printf("[TRACE] ModuleInstaller: also discarding downstream %s", subKey) } delete(manifest, subKey) } @@ -147,10 +149,10 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo // Clean up any stale cache directory that might be present. // If this is a local (relative) source then the dir will // not exist, but we'll ignore that. - log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key) + log.Printf("[TRACE] ModuleInstaller: cleaning directory %s prior to install of %s", instPath, key) err := os.RemoveAll(instPath) if err != nil && !os.IsNotExist(err) { - log.Printf("[TRACE] failed to remove %s: %s", key, err) + log.Printf("[TRACE] ModuleInstaller: failed to remove %s: %s", key, err) diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Failed to remove local module cache", @@ -170,7 +172,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo mod, mDiags := earlyconfig.LoadModule(record.Dir) diags = diags.Append(mDiags) - log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir) + log.Printf("[TRACE] ModuleInstaller: Module installer: %s %s already installed in %s", key, record.Version, record.Dir) return mod, record.Version, diags } } @@ -181,7 +183,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo switch { case isLocalSourceAddr(req.SourceAddr): - log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr) + log.Printf("[TRACE] ModuleInstaller: %s has local path %q", key, req.SourceAddr) mod, mDiags := i.installLocalModule(req, key, manifest, hooks) diags = append(diags, mDiags...) return mod, nil, diags @@ -192,14 +194,14 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo // Should never happen because isRegistrySourceAddr already validated panic(err) } - log.Printf("[TRACE] %s is a registry module at %s", key, addr) + log.Printf("[TRACE] ModuleInstaller: %s is a registry module at %s", key, addr) mod, v, mDiags := i.installRegistryModule(req, key, instPath, addr, manifest, hooks, getter) diags = append(diags, mDiags...) return mod, v, diags default: - log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr) + log.Printf("[TRACE] ModuleInstaller: %s address %q will be handled by go-getter", key, req.SourceAddr) mod, mDiags := i.installGoGetterModule(req, key, instPath, manifest, hooks, getter) diags = append(diags, mDiags...) @@ -222,7 +224,7 @@ func (i *ModuleInstaller) installDescendentModules(rootMod *tfconfig.Module, roo return diags } -func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks InstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) { +func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key string, manifest modsdir.Manifest, hooks ModuleInstallHooks) (*tfconfig.Module, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics parentKey := manifest.ModuleKey(req.Parent.Path) @@ -244,7 +246,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key // filesystem at all because the parent already wrote // the files we need, and so we just load up what's already here. newDir := filepath.Join(parentRecord.Dir, req.SourceAddr) - log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir) + log.Printf("[TRACE] ModuleInstaller: %s uses directory from parent: %s", key, newDir) mod, mDiags := earlyconfig.LoadModule(newDir) if mod == nil { // nil indicates missing or unreadable directory, so we'll @@ -271,7 +273,7 @@ func (i *ModuleInstaller) installLocalModule(req *earlyconfig.ModuleRequest, key return mod, diags } -func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) { +func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, key string, instPath string, addr *regsrc.Module, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, *version.Version, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics hostname, err := addr.SvcHost() @@ -353,7 +355,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, // If we've found a pre-release version then we'll ignore it unless // it was exactly requested. if v.Prerelease() != "" && req.VersionConstraints.String() != v.String() { - log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v) + log.Printf("[TRACE] ModuleInstaller: %s ignoring %s because it is a pre-release and was not requested exactly", key, v) continue } @@ -404,7 +406,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, return nil, nil, diags } - log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr) + log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr) modDir, err := getter.getWithGoGetter(instPath, dlAddr) if err != nil { @@ -421,7 +423,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, return nil, nil, diags } - log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir) + log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, dlAddr, modDir) if addr.RawSubmodule != "" { // Append the user's requested subdirectory to any subdirectory that @@ -429,7 +431,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, modDir = filepath.Join(modDir, addr.RawSubmodule) } - log.Printf("[TRACE] %s should now be at %s", key, modDir) + log.Printf("[TRACE] ModuleInstaller: %s should now be at %s", key, modDir) // Finally we are ready to try actually loading the module. mod, mDiags := earlyconfig.LoadModule(modDir) @@ -461,7 +463,7 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, return mod, latestMatch, diags } -func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks InstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) { +func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, key string, instPath string, manifest modsdir.Manifest, hooks ModuleInstallHooks, getter reusingGetter) (*tfconfig.Module, tfdiags.Diagnostics) { var diags tfdiags.Diagnostics // Report up to the caller that we're about to start downloading. @@ -483,7 +485,7 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, return nil, diags } - log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir) + log.Printf("[TRACE] ModuleInstaller: %s %q was downloaded to %s", key, req.SourceAddr, modDir) mod, mDiags := earlyconfig.LoadModule(modDir) if mod == nil { diff --git a/internal/init/module_install_hooks.go b/internal/initwd/module_install_hooks.go similarity index 63% rename from internal/init/module_install_hooks.go rename to internal/initwd/module_install_hooks.go index 94135abe3..817a6dc83 100644 --- a/internal/init/module_install_hooks.go +++ b/internal/initwd/module_install_hooks.go @@ -1,16 +1,16 @@ -package init +package initwd import ( version "github.com/hashicorp/go-version" ) -// InstallHooks is an interface used to provide notifications about the +// ModuleInstallHooks is an interface used to provide notifications about the // installation process being orchestrated by InstallModules. // // This interface may have new methods added in future, so implementers should // embed InstallHooksImpl to get no-op implementations of any unimplemented // methods. -type InstallHooks interface { +type ModuleInstallHooks interface { // Download is called for modules that are retrieved from a remote source // before that download begins, to allow a caller to give feedback // on progress through a possibly-long sequence of downloads. @@ -21,16 +21,16 @@ type InstallHooks interface { Install(moduleAddr string, version *version.Version, localPath string) } -// InstallHooksImpl is a do-nothing implementation of InstallHooks that +// ModuleInstallHooksImpl is a do-nothing implementation of InstallHooks that // can be embedded in another implementation struct to allow only partial // implementation of the interface. -type InstallHooksImpl struct { +type ModuleInstallHooksImpl struct { } -func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) { +func (h ModuleInstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) { } -func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) { +func (h ModuleInstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) { } -var _ InstallHooks = InstallHooksImpl{} +var _ ModuleInstallHooks = ModuleInstallHooksImpl{} diff --git a/configs/configload/loader_install_test.go b/internal/initwd/module_install_test.go similarity index 68% rename from configs/configload/loader_install_test.go rename to internal/initwd/module_install_test.go index e0e6bdf98..4cce4071a 100644 --- a/configs/configload/loader_install_test.go +++ b/internal/initwd/module_install_test.go @@ -1,23 +1,47 @@ -package configload +package initwd import ( + "flag" + "fmt" + "io/ioutil" + "log" "os" "path/filepath" "strings" "testing" + "github.com/go-test/deep" version "github.com/hashicorp/go-version" "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/helper/logging" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/tfdiags" ) -func TestLoaderInstallModules_local(t *testing.T) { +func TestMain(m *testing.M) { + flag.Parse() + if testing.Verbose() { + // if we're verbose, use the logging requested by TF_LOG + logging.SetOutput() + } else { + // otherwise silence all logs + log.SetOutput(ioutil.Discard) + } + + os.Exit(m.Run()) +} + +func TestModuleInstaller(t *testing.T) { fixtureDir := filepath.Clean("test-fixtures/local-modules") - loader, done := tempChdirLoader(t, fixtureDir) + dir, done := tempChdir(t, fixtureDir) defer done() hooks := &testInstallHooks{} - diags := loader.InstallModules(".", false, hooks) + modulesDir := filepath.Join(dir, ".terraform/modules") + inst := NewModuleInstaller(modulesDir, nil) + diags := inst.InstallModules(".", false, hooks) assertNoDiagnostics(t, diags) wantCalls := []testInstallHookCall{ @@ -39,10 +63,17 @@ func TestLoaderInstallModules_local(t *testing.T) { return } + loader, err := configload.NewLoader(&configload.Config{ + ModulesDir: modulesDir, + }) + if err != nil { + t.Fatal(err) + } + // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) config, loadDiags := loader.LoadConfig(".") - assertNoDiagnostics(t, loadDiags) + assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) wantTraces := map[string]string{ "": "in root module", @@ -67,13 +98,14 @@ func TestLoaderInstallModules_registry(t *testing.T) { t.Skip("this test accesses registry.terraform.io and github.com; set TF_ACC=1 to run it") } - fixtureDir := filepath.Clean("test-fixtures/registry-modules") - loader, done := tempChdirLoader(t, fixtureDir) + fixtureDir := filepath.Clean("test-fixtures/local-modules") + dir, done := tempChdir(t, fixtureDir) defer done() hooks := &testInstallHooks{} - - diags := loader.InstallModules(".", false, hooks) + modulesDir := filepath.Join(dir, ".terraform/modules") + inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil)) + diags := inst.InstallModules(dir, false, hooks) assertNoDiagnostics(t, diags) v := version.Must(version.NewVersion("0.0.1")) @@ -153,10 +185,17 @@ func TestLoaderInstallModules_registry(t *testing.T) { return } + loader, err := configload.NewLoader(&configload.Config{ + ModulesDir: modulesDir, + }) + if err != nil { + t.Fatal(err) + } + // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) config, loadDiags := loader.LoadConfig(".") - assertNoDiagnostics(t, loadDiags) + assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) wantTraces := map[string]string{ "": "in local caller for registry-modules", @@ -187,12 +226,13 @@ func TestLoaderInstallModules_goGetter(t *testing.T) { } fixtureDir := filepath.Clean("test-fixtures/go-getter-modules") - loader, done := tempChdirLoader(t, fixtureDir) + dir, done := tempChdir(t, fixtureDir) defer done() hooks := &testInstallHooks{} - - diags := loader.InstallModules(".", false, hooks) + modulesDir := filepath.Join(dir, ".terraform/modules") + inst := NewModuleInstaller(modulesDir, registry.NewClient(nil, nil)) + diags := inst.InstallModules(dir, false, hooks) assertNoDiagnostics(t, diags) wantCalls := []testInstallHookCall{ @@ -264,10 +304,17 @@ func TestLoaderInstallModules_goGetter(t *testing.T) { return } + loader, err := configload.NewLoader(&configload.Config{ + ModulesDir: modulesDir, + }) + if err != nil { + t.Fatal(err) + } + // Make sure the configuration is loadable now. // (This ensures that correct information is recorded in the manifest.) config, loadDiags := loader.LoadConfig(".") - assertNoDiagnostics(t, loadDiags) + assertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags)) wantTraces := map[string]string{ "": "in local caller for go-getter-modules", @@ -321,3 +368,97 @@ func (h *testInstallHooks) Install(moduleAddr string, version *version.Version, LocalPath: localPath, }) } + +// tempChdir copies the contents of the given directory to a temporary +// directory and changes the test process's current working directory to +// point to that directory. Also returned is a function that should be +// called at the end of the test (e.g. via "defer") to restore the previous +// working directory. +// +// Tests using this helper cannot safely be run in parallel with other tests. +func tempChdir(t *testing.T, sourceDir string) (string, func()) { + t.Helper() + + tmpDir, err := ioutil.TempDir("", "terraform-configload") + if err != nil { + t.Fatalf("failed to create temporary directory: %s", err) + return "", nil + } + + if err := copyDir(tmpDir, sourceDir); err != nil { + t.Fatalf("failed to copy fixture to temporary directory: %s", err) + return "", nil + } + + oldDir, err := os.Getwd() + if err != nil { + t.Fatalf("failed to determine current working directory: %s", err) + return "", nil + } + + err = os.Chdir(tmpDir) + if err != nil { + t.Fatalf("failed to switch to temp dir %s: %s", tmpDir, err) + return "", nil + } + + // Most of the tests need this, so we'll make it just in case. + os.MkdirAll(filepath.Join(tmpDir, ".terraform/modules"), os.ModePerm) + + t.Logf("tempChdir switched to %s after copying from %s", tmpDir, sourceDir) + + return tmpDir, func() { + err := os.Chdir(oldDir) + if err != nil { + panic(fmt.Errorf("failed to restore previous working directory %s: %s", oldDir, err)) + } + + if os.Getenv("TF_CONFIGLOAD_TEST_KEEP_TMP") == "" { + os.RemoveAll(tmpDir) + } + } +} + +func assertNoDiagnostics(t *testing.T, diags tfdiags.Diagnostics) bool { + t.Helper() + return assertDiagnosticCount(t, diags, 0) +} + +func assertDiagnosticCount(t *testing.T, diags tfdiags.Diagnostics, want int) bool { + t.Helper() + if len(diags) != 0 { + t.Errorf("wrong number of diagnostics %d; want %d", len(diags), want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true + } + return false +} + +func assertDiagnosticSummary(t *testing.T, diags tfdiags.Diagnostics, want string) bool { + t.Helper() + + for _, diag := range diags { + if diag.Description().Summary == want { + return false + } + } + + t.Errorf("missing diagnostic summary %q", want) + for _, diag := range diags { + t.Logf("- %s", diag) + } + return true +} + +func assertResultDeepEqual(t *testing.T, got, want interface{}) bool { + t.Helper() + if diff := deep.Equal(got, want); diff != nil { + for _, problem := range diff { + t.Errorf("%s", problem) + } + return true + } + return false +} diff --git a/internal/initwd/test-fixtures/already-installed/root.tf b/internal/initwd/test-fixtures/already-installed/root.tf new file mode 100644 index 000000000..8a4473942 --- /dev/null +++ b/internal/initwd/test-fixtures/already-installed/root.tf @@ -0,0 +1,10 @@ + +module "child_a" { + source = "example.com/foo/bar_a/baz" + version = ">= 1.0.0" +} + +module "child_b" { + source = "example.com/foo/bar_b/baz" + version = ">= 1.0.0" +} diff --git a/internal/initwd/test-fixtures/empty/.gitignore b/internal/initwd/test-fixtures/empty/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/internal/initwd/test-fixtures/go-getter-modules/.gitignore b/internal/initwd/test-fixtures/go-getter-modules/.gitignore new file mode 100644 index 000000000..6e0db03a8 --- /dev/null +++ b/internal/initwd/test-fixtures/go-getter-modules/.gitignore @@ -0,0 +1 @@ +.terraform/* diff --git a/internal/initwd/test-fixtures/go-getter-modules/root.tf b/internal/initwd/test-fixtures/go-getter-modules/root.tf new file mode 100644 index 000000000..9b174a7a5 --- /dev/null +++ b/internal/initwd/test-fixtures/go-getter-modules/root.tf @@ -0,0 +1,21 @@ +# This fixture depends on a github repo at: +# https://github.com/hashicorp/terraform-aws-module-installer-acctest +# ...and expects its v0.0.1 tag to be pointing at the following commit: +# d676ab2559d4e0621d59e3c3c4cbb33958ac4608 + +variable "v" { + description = "in local caller for go-getter-modules" + default = "" +} + +module "acctest_root" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest?ref=v0.0.1" +} + +module "acctest_child_a" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_a?ref=v0.0.1" +} + +module "acctest_child_b" { + source = "github.com/hashicorp/terraform-aws-module-installer-acctest//modules/child_b?ref=v0.0.1" +} diff --git a/internal/initwd/test-fixtures/local-modules/child_a/child_a.tf b/internal/initwd/test-fixtures/local-modules/child_a/child_a.tf new file mode 100644 index 000000000..68ebb8e40 --- /dev/null +++ b/internal/initwd/test-fixtures/local-modules/child_a/child_a.tf @@ -0,0 +1,9 @@ + +variable "v" { + description = "in child_a module" + default = "" +} + +module "child_b" { + source = "./child_b" +} diff --git a/internal/initwd/test-fixtures/local-modules/child_a/child_b/child_b.tf b/internal/initwd/test-fixtures/local-modules/child_a/child_b/child_b.tf new file mode 100644 index 000000000..e2e220916 --- /dev/null +++ b/internal/initwd/test-fixtures/local-modules/child_a/child_b/child_b.tf @@ -0,0 +1,9 @@ + +variable "v" { + description = "in child_b module" + default = "" +} + +output "hello" { + value = "Hello from child_b!" +} diff --git a/internal/initwd/test-fixtures/local-modules/root.tf b/internal/initwd/test-fixtures/local-modules/root.tf new file mode 100644 index 000000000..3b4c6416d --- /dev/null +++ b/internal/initwd/test-fixtures/local-modules/root.tf @@ -0,0 +1,9 @@ + +variable "v" { + description = "in root module" + default = "" +} + +module "child_a" { + source = "./child_a" +} diff --git a/internal/initwd/test-fixtures/registry-modules/.gitignore b/internal/initwd/test-fixtures/registry-modules/.gitignore new file mode 100644 index 000000000..6e0db03a8 --- /dev/null +++ b/internal/initwd/test-fixtures/registry-modules/.gitignore @@ -0,0 +1 @@ +.terraform/* diff --git a/internal/initwd/test-fixtures/registry-modules/root.tf b/internal/initwd/test-fixtures/registry-modules/root.tf new file mode 100644 index 000000000..4b5ad1f1e --- /dev/null +++ b/internal/initwd/test-fixtures/registry-modules/root.tf @@ -0,0 +1,33 @@ +# This fixture indirectly depends on a github repo at: +# https://github.com/hashicorp/terraform-aws-module-installer-acctest +# ...and expects its v0.0.1 tag to be pointing at the following commit: +# d676ab2559d4e0621d59e3c3c4cbb33958ac4608 +# +# This repository is accessed indirectly via: +# https://registry.terraform.io/modules/hashicorp/module-installer-acctest/aws/0.0.1 +# +# Since the tag's id is included in a downloaded archive, it is expected to +# have the following id: +# 853d03855b3290a3ca491d4c3a7684572dd42237 +# (this particular assumption is encoded in the tests that use this fixture) + + +variable "v" { + description = "in local caller for registry-modules" + default = "" +} + +module "acctest_root" { + source = "hashicorp/module-installer-acctest/aws" + version = "0.0.1" +} + +module "acctest_child_a" { + source = "hashicorp/module-installer-acctest/aws//modules/child_a" + version = "0.0.1" +} + +module "acctest_child_b" { + source = "hashicorp/module-installer-acctest/aws//modules/child_b" + version = "0.0.1" +} diff --git a/internal/initwd/testing.go b/internal/initwd/testing.go new file mode 100644 index 000000000..072d42a10 --- /dev/null +++ b/internal/initwd/testing.go @@ -0,0 +1,67 @@ +package initwd + +import ( + "github.com/hashicorp/terraform/registry" + "testing" + + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/configs/configload" + "github.com/hashicorp/terraform/tfdiags" +) + +// LoadConfigForTests is a convenience wrapper around configload.NewLoaderForTests, +// ModuleInstaller.InstallModules and configload.Loader.LoadConfig that allows +// a test configuration to be loaded in a single step. +// +// If module installation fails, t.Fatal (or similar) is called to halt +// execution of the test, under the assumption that installation failures are +// not expected. If installation failures _are_ expected then use +// NewLoaderForTests and work with the loader object directly. If module +// installation succeeds but generates warnings, these warnings are discarded. +// +// If installation succeeds but errors are detected during loading then a +// possibly-incomplete config is returned along with error diagnostics. The +// test run is not aborted in this case, so that the caller can make assertions +// against the returned diagnostics. +// +// As with NewLoaderForTests, a cleanup function is returned which must be +// called before the test completes in order to remove the temporary +// modules directory. +func LoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func(), tfdiags.Diagnostics) { + t.Helper() + + var diags tfdiags.Diagnostics + + loader, cleanup := configload.NewLoaderForTests(t) + inst := NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil)) + + moreDiags := inst.InstallModules(rootDir, true, ModuleInstallHooksImpl{}) + diags = diags.Append(moreDiags) + if diags.HasErrors() { + cleanup() + t.Fatal(diags.Err()) + return nil, nil, func() {}, diags + } + + config, hclDiags := loader.LoadConfig(rootDir) + diags = diags.Append(hclDiags) + return config, loader, cleanup, diags +} + +// MustLoadConfigForTests is a variant of LoadConfigForTests which calls +// t.Fatal (or similar) if there are any errors during loading, and thus +// does not return diagnostics at all. +// +// This is useful for concisely writing tests that don't expect errors at +// all. For tests that expect errors and need to assert against them, use +// LoadConfigForTests instead. +func MustLoadConfigForTests(t *testing.T, rootDir string) (*configs.Config, *configload.Loader, func()) { + t.Helper() + + config, loader, cleanup, diags := LoadConfigForTests(t, rootDir) + if diags.HasErrors() { + cleanup() + t.Fatal(diags.Err()) + } + return config, loader, cleanup +} diff --git a/internal/modsdir/manifest.go b/internal/modsdir/manifest.go index 09c45017c..36f6c033f 100644 --- a/internal/modsdir/manifest.go +++ b/internal/modsdir/manifest.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "os" "path/filepath" @@ -95,6 +96,9 @@ func ReadManifestSnapshotForDir(dir string) (Manifest, error) { fn := filepath.Join(dir, ManifestSnapshotFilename) r, err := os.Open(fn) if err != nil { + if os.IsNotExist(err) { + return make(Manifest), nil // missing file is okay and treated as empty + } return nil, err } return ReadManifestSnapshot(r) @@ -125,6 +129,7 @@ func (m Manifest) WriteSnapshot(w io.Writer) error { func (m Manifest) WriteSnapshotToDir(dir string) error { fn := filepath.Join(dir, ManifestSnapshotFilename) + log.Printf("[TRACE] modsdir: writing modules manifest to %s", fn) w, err := os.Create(fn) if err != nil { return err diff --git a/repl/session_test.go b/repl/session_test.go index dc560bf55..425fcdc0f 100644 --- a/repl/session_test.go +++ b/repl/session_test.go @@ -2,6 +2,7 @@ package repl import ( "flag" + "github.com/hashicorp/terraform/internal/initwd" "io/ioutil" "log" "os" @@ -11,7 +12,6 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/hashicorp/terraform/addrs" - "github.com/hashicorp/terraform/configs/configload" "github.com/hashicorp/terraform/configs/configschema" "github.com/hashicorp/terraform/helper/logging" "github.com/hashicorp/terraform/providers" @@ -202,17 +202,10 @@ func testSession(t *testing.T, test testSessionTest) { }, } - loader, cleanup := configload.NewLoaderForTests(t) + config, _, cleanup, configDiags := initwd.LoadConfigForTests(t, "testdata/config-fixture") defer cleanup() - - configDiags := loader.InstallModules("testdata/config-fixture", false, nil) if configDiags.HasErrors() { - t.Fatalf("unexpected problems initializing test config: %s", configDiags.Error()) - } - - config, configDiags := loader.LoadConfig("testdata/config-fixture") - if configDiags.HasErrors() { - t.Fatalf("unexpected problems loading config: %s", configDiags.Error()) + t.Fatalf("unexpected problems loading config: %s", configDiags.Err()) } // Build the TF context diff --git a/terraform/terraform_test.go b/terraform/terraform_test.go index 650ebe8c6..2407bf18f 100644 --- a/terraform/terraform_test.go +++ b/terraform/terraform_test.go @@ -2,6 +2,8 @@ package terraform import ( "flag" + "github.com/hashicorp/terraform/internal/initwd" + "github.com/hashicorp/terraform/registry" "io" "io/ioutil" "log" @@ -102,7 +104,6 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config t.Helper() dir := filepath.Join(fixtureDir, name) - // FIXME: We're not dealing with the cleanup function here because // this testModule function is used all over and so we don't want to // change its interface at this late stage. @@ -111,9 +112,10 @@ func testModuleWithSnapshot(t *testing.T, name string) (*configs.Config, *config // Test modules usually do not refer to remote sources, and for local // sources only this ultimately just records all of the module paths // in a JSON file so that we can load them below. - diags := loader.InstallModules(dir, true, configload.InstallHooksImpl{}) - if diags.HasErrors() { - t.Fatal(diags.Error()) + inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil)) + instDiags := inst.InstallModules(dir, true, initwd.ModuleInstallHooksImpl{}) + if instDiags.HasErrors() { + t.Fatal(instDiags.Err()) } config, snap, diags := loader.LoadConfigWithSnapshot(dir) @@ -162,9 +164,10 @@ func testModuleInline(t *testing.T, sources map[string]string) *configs.Config { // Test modules usually do not refer to remote sources, and for local // sources only this ultimately just records all of the module paths // in a JSON file so that we can load them below. - diags := loader.InstallModules(cfgPath, true, configload.InstallHooksImpl{}) - if diags.HasErrors() { - t.Fatal(diags.Error()) + inst := initwd.NewModuleInstaller(loader.ModulesDir(), registry.NewClient(nil, nil)) + instDiags := inst.InstallModules(cfgPath, true, initwd.ModuleInstallHooksImpl{}) + if instDiags.HasErrors() { + t.Fatal(instDiags.Err()) } config, diags := loader.LoadConfig(cfgPath)