From 92d6a30bb4e8fbad0968a9915c6d90435a4a08f6 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 15 Apr 2020 11:48:24 -0700 Subject: [PATCH] main: skip direct provider installation for providers available locally This more closely replicates the 0.12-and-earlier behavior, where having at least one version of a provider installed locally would totally disable any attempt to look for newer versions remotely. This is just for the implicit default behavior. Assumption is that later we'll have an explicit configuration mechanism that will allow the user to specify exactly where to look for what, and thus avoid tricky heuristics like this. --- command/e2etest/init_test.go | 43 +++++++++++++++ .../testdata/local-only-provider/main.tf | 21 +++++++ .../terraform-provider-happycloud_v1.2.0 | 2 + provider_source.go | 55 +++++++++++++++++-- 4 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 command/e2etest/testdata/local-only-provider/main.tf create mode 100644 command/e2etest/testdata/local-only-provider/terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go index ca618fdce..c1ad00805 100644 --- a/command/e2etest/init_test.go +++ b/command/e2etest/init_test.go @@ -130,6 +130,49 @@ func TestInitProvidersVendored(t *testing.T) { } +func TestInitProvidersLocalOnly(t *testing.T) { + t.Parallel() + + // This test should not reach out to the network if it is behaving as + // intended. If it _does_ try to access an upstream registry and encounter + // an error doing so then that's a legitimate test failure that should be + // fixed. (If it incorrectly reaches out anywhere then it's likely to be + // to the host "example.com", which is the placeholder domain we use in + // the test fixture.) + + fixturePath := filepath.Join("testdata", "local-only-provider") + tf := e2e.NewBinary(terraformBin, fixturePath) + defer tf.Close() + + // Our fixture dir has a generic os_arch dir, which we need to customize + // to the actual OS/arch where this test is running in order to get the + // desired result. + fixtMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch") + wantMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + err := os.Rename(fixtMachineDir, wantMachineDir) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + if !strings.Contains(stdout, "Terraform has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") { + t.Errorf("provider download message is missing from output:\n%s", stdout) + t.Logf("(this can happen if you have a conflicting copy of the plugin in one of the global plugin search dirs)") + } +} + func TestInitProviders_pluginCache(t *testing.T) { t.Parallel() diff --git a/command/e2etest/testdata/local-only-provider/main.tf b/command/e2etest/testdata/local-only-provider/main.tf new file mode 100644 index 000000000..a521cf07b --- /dev/null +++ b/command/e2etest/testdata/local-only-provider/main.tf @@ -0,0 +1,21 @@ +# The purpose of this test is to refer to a provider whose address contains +# a hostname that is only used for namespacing purposes and doesn't actually +# have a provider registry deployed at it. +# +# A user can install such a provider in one of the implied local filesystem +# directories and Terraform should accept that as the selection for that +# provider without producing any errors about the fact that example.com +# does not have a provider registry. +# +# For this test in particular we're using the "vendor" directory that is +# the documented way to include provider plugins directly inside a +# configuration uploaded to Terraform Cloud, but this functionality applies +# to all of the implicit local filesystem search directories. + +terraform { + required_providers { + happycloud = { + source = "example.com/awesomecorp/happycloud" + } + } +} diff --git a/command/e2etest/testdata/local-only-provider/terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 b/command/e2etest/testdata/local-only-provider/terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 new file mode 100644 index 000000000..3299bec8a --- /dev/null +++ b/command/e2etest/testdata/local-only-provider/terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 @@ -0,0 +1,2 @@ +This is not a real plugin executable. It's just here to be discovered by the +provider installation process. diff --git a/provider_source.go b/provider_source.go index 9524e0985..8e91658a8 100644 --- a/provider_source.go +++ b/provider_source.go @@ -6,8 +6,9 @@ import ( "path/filepath" "github.com/apparentlymart/go-userdirs/userdirs" - "github.com/hashicorp/terraform-svchost/disco" + + "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/command/cliconfig" "github.com/hashicorp/terraform/internal/getproviders" ) @@ -19,8 +20,21 @@ import ( func providerSource(services *disco.Disco) getproviders.Source { // We're not yet using the CLI config here because we've not implemented // yet the new configuration constructs to customize provider search - // locations. That'll come later. - // For now, we have a fixed set of search directories: + // locations. That'll come later. For now, we just always use the + // implicit default provider source. + return implicitProviderSource(services) +} + +// implicitProviderSource builds a default provider source to use if there's +// no explicit provider installation configuration in the CLI config. +// +// This implicit source looks in a number of local filesystem directories and +// directly in a provider's upstream registry. Any providers that have at least +// one version available in a local directory are implicitly excluded from +// direct installation, as if the user had listed them explicitly in the +// "exclude" argument in the direct provider source in the CLI config. +func implicitProviderSource(services *disco.Disco) getproviders.Source { + // The local search directories we use for implicit configuration are: // - The "terraform.d/plugins" directory in the current working directory, // which we've historically documented as a place to put plugins as a // way to include them in bundles uploaded to Terraform Cloud, where @@ -31,10 +45,17 @@ func providerSource(services *disco.Disco) getproviders.Source { // following e.g. the XDG base directory specification on Unix systems, // Apple's guidelines on OS X, and "known folders" on Windows. // - // Those directories are checked in addition to the direct upstream - // registry specified in the provider's address. + // Any provider we find in one of those implicit directories will be + // automatically excluded from direct installation from an upstream + // registry. Anything not available locally will query its primary + // upstream registry. var searchRules []getproviders.MultiSourceSelector + // We'll track any providers we can find in the local search directories + // along the way, and then exclude them from the registry source we'll + // finally add at the end. + foundLocally := map[addrs.Provider]struct{}{} + addLocalDir := func(dir string) { // We'll make sure the directory actually exists before we add it, // because otherwise installation would always fail trying to look @@ -44,9 +65,23 @@ func providerSource(services *disco.Disco) getproviders.Source { // don't exist to help users get their configurations right.) if info, err := os.Stat(dir); err == nil && info.IsDir() { log.Printf("[DEBUG] will search for provider plugins in %s", dir) + fsSource := getproviders.NewFilesystemMirrorSource(dir) + + // We'll peep into the source to find out what providers it seems + // to be providing, so that we can exclude those from direct + // install. This might fail, in which case we'll just silently + // ignore it and assume it would fail during installation later too + // and therefore effectively doesn't provide _any_ packages. + if available, err := fsSource.AllAvailablePackages(); err == nil { + for found := range available { + foundLocally[found] = struct{}{} + } + } + searchRules = append(searchRules, getproviders.MultiSourceSelector{ - Source: getproviders.NewFilesystemMirrorSource(dir), + Source: fsSource, }) + } else { log.Printf("[DEBUG] ignoring non-existing provider search directory %s", dir) } @@ -72,6 +107,13 @@ func providerSource(services *disco.Disco) getproviders.Source { addLocalDir(dir) } + // Anything we found in local directories above is excluded from being + // looked up via the registry source we're about to construct. + var directExcluded getproviders.MultiSourceMatchingPatterns + for addr := range foundLocally { + directExcluded = append(directExcluded, addr) + } + // Last but not least, the main registry source! We'll wrap a caching // layer around this one to help optimize the several network requests // we'll end up making to it while treating it as one of several sources @@ -83,6 +125,7 @@ func providerSource(services *disco.Disco) getproviders.Source { Source: getproviders.NewMemoizeSource( getproviders.NewRegistrySource(services), ), + Exclude: directExcluded, }) return getproviders.MultiSource(searchRules)