From f753974bb3ad001664596b8ba811f899d1835015 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Mon, 12 Jun 2017 18:22:47 -0700 Subject: [PATCH] plugin/discovery: Installer interface, and provider implementation Previously we had a "getProvider" function type used to implement plugin fetching. Here we replace that with an interface type, initially with just a "Get" function. For now this just simplifies the interface by allowing the target directory and protocol version to be members of the struct rather than passed as arguments. A later change will extend this interface to also include a method to purge unused plugins, so that upgrading frequently doesn't leave behind a trail of unused executable files. --- command/init.go | 25 ++++++------ command/init_test.go | 66 +++++++++++++++++++------------- command/plugins.go | 36 ++++++++++++++++-- command/plugins_test.go | 60 +++++++++++++++++++++-------- plugin/discovery/get.go | 73 ++++++++++++++++++++++++++++++------ plugin/discovery/get_test.go | 38 ++++++++++++++----- plugin/discovery/meta_set.go | 18 +++++++++ plugin/discovery/version.go | 4 ++ 8 files changed, 242 insertions(+), 78 deletions(-) diff --git a/command/init.go b/command/init.go index 2cce24e06..c2fab4811 100644 --- a/command/init.go +++ b/command/init.go @@ -23,11 +23,11 @@ import ( type InitCommand struct { Meta - // getProvider fetches providers that aren't found locally, and unpacks - // them into the dst directory. - // This uses discovery.GetProvider by default, but it provided here as a - // way to mock fetching providers for tests. - getProvider func(dst, provider string, req discovery.Constraints, protoVersion uint) error + // providerInstaller is used to download and install providers that + // aren't found locally. This uses a discovery.ProviderInstaller instance + // by default, but it can be overridden here as a way to mock fetching + // providers for tests. + providerInstaller discovery.Installer } func (c *InitCommand) Run(args []string) int { @@ -52,8 +52,12 @@ func (c *InitCommand) Run(args []string) int { } // set getProvider if we don't have a test version already - if c.getProvider == nil { - c.getProvider = discovery.GetProvider + if c.providerInstaller == nil { + c.providerInstaller = &discovery.ProviderInstaller{ + Dir: c.pluginDir(), + + PluginProtocolVersion: plugin.Handshake.ProtocolVersion, + } } // Validate the arg count @@ -176,7 +180,7 @@ func (c *InitCommand) Run(args []string) int { "[reset][bold]Initializing provider plugins...", )) - err = c.getProviders(path, sMgr.State()) + err = c.getProviders(path, sMgr.State(), flagUpgrade) if err != nil { // this function provides its own output log.Printf("[ERROR] %s", err) @@ -197,7 +201,7 @@ func (c *InitCommand) Run(args []string) int { // Load the complete module tree, and fetch any missing providers. // This method outputs its own Ui. -func (c *InitCommand) getProviders(path string, state *terraform.State) error { +func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error { mod, err := c.Module(path) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting plugins: %s", err)) @@ -213,11 +217,10 @@ func (c *InitCommand) getProviders(path string, state *terraform.State) error { requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements() missing := c.missingPlugins(available, requirements) - dst := c.pluginDir() var errs error for provider, reqd := range missing { c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider)) - err := c.getProvider(dst, provider, reqd.Versions, plugin.Handshake.ProtocolVersion) + _, err := c.providerInstaller.Get(provider, reqd.Versions) if err != nil { c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions)) diff --git a/command/init_test.go b/command/init_test.go index de4f3c1db..e5a548717 100644 --- a/command/init_test.go +++ b/command/init_test.go @@ -448,7 +448,13 @@ func TestInit_getProvider(t *testing.T) { defer os.RemoveAll(td) defer testChdir(t, td)() - getter := &mockGetProvider{ + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + installer := &mockProviderInstaller{ Providers: map[string][]string{ // looking for an exact version "exact": []string{"1.2.3"}, @@ -457,15 +463,13 @@ func TestInit_getProvider(t *testing.T) { // config specifies "between": []string{"3.4.5", "2.3.4", "1.2.3"}, }, + + Dir: m.pluginDir(), } - ui := new(cli.MockUi) c := &InitCommand{ - Meta: Meta{ - testingOverrides: metaOverridesForProvider(testProvider()), - Ui: ui, - }, - getProvider: getter.GetProvider, + Meta: m, + providerInstaller: installer, } args := []string{} @@ -474,15 +478,15 @@ func TestInit_getProvider(t *testing.T) { } // check that we got the providers for our config - exactPath := filepath.Join(c.pluginDir(), getter.FileName("exact", "1.2.3")) + exactPath := filepath.Join(c.pluginDir(), installer.FileName("exact", "1.2.3")) if _, err := os.Stat(exactPath); os.IsNotExist(err) { t.Fatal("provider 'exact' not downloaded") } - greaterThanPath := filepath.Join(c.pluginDir(), getter.FileName("greater_than", "2.3.4")) + greaterThanPath := filepath.Join(c.pluginDir(), installer.FileName("greater_than", "2.3.4")) if _, err := os.Stat(greaterThanPath); os.IsNotExist(err) { t.Fatal("provider 'greater_than' not downloaded") } - betweenPath := filepath.Join(c.pluginDir(), getter.FileName("between", "2.3.4")) + betweenPath := filepath.Join(c.pluginDir(), installer.FileName("between", "2.3.4")) if _, err := os.Stat(betweenPath); os.IsNotExist(err) { t.Fatal("provider 'between' not downloaded") } @@ -495,7 +499,13 @@ func TestInit_getProviderMissing(t *testing.T) { defer os.RemoveAll(td) defer testChdir(t, td)() - getter := &mockGetProvider{ + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + installer := &mockProviderInstaller{ Providers: map[string][]string{ // looking for exact version 1.2.3 "exact": []string{"1.2.4"}, @@ -504,15 +514,13 @@ func TestInit_getProviderMissing(t *testing.T) { // config specifies "between": []string{"3.4.5", "2.3.4", "1.2.3"}, }, + + Dir: m.pluginDir(), } - ui := new(cli.MockUi) c := &InitCommand{ - Meta: Meta{ - testingOverrides: metaOverridesForProvider(testProvider()), - Ui: ui, - }, - getProvider: getter.GetProvider, + Meta: m, + providerInstaller: installer, } args := []string{} @@ -544,9 +552,9 @@ func TestInit_getProviderHaveLegacyVersion(t *testing.T) { testingOverrides: metaOverridesForProvider(testProvider()), Ui: ui, }, - getProvider: func(dst, provider string, req discovery.Constraints, protoVersion uint) error { - return fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider) - }, + providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) { + return discovery.PluginMeta{}, fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider) + }), } args := []string{} @@ -566,19 +574,23 @@ func TestInit_providerLockFile(t *testing.T) { defer os.RemoveAll(td) defer testChdir(t, td)() - getter := &mockGetProvider{ + ui := new(cli.MockUi) + m := Meta{ + testingOverrides: metaOverridesForProvider(testProvider()), + Ui: ui, + } + + installer := &mockProviderInstaller{ Providers: map[string][]string{ "test": []string{"1.2.3"}, }, + + Dir: m.pluginDir(), } - ui := new(cli.MockUi) c := &InitCommand{ - Meta: Meta{ - testingOverrides: metaOverridesForProvider(testProvider()), - Ui: ui, - }, - getProvider: getter.GetProvider, + Meta: m, + providerInstaller: installer, } args := []string{} diff --git a/command/plugins.go b/command/plugins.go index 42d224ded..78d07c92f 100644 --- a/command/plugins.go +++ b/command/plugins.go @@ -79,7 +79,7 @@ func (m *Meta) pluginDir() string { // Earlier entries in this slice get priority over later when multiple copies // of the same plugin version are found, but newer versions always override // older versions where both satisfy the provider version constraints. -func (m *Meta) pluginDirs() []string { +func (m *Meta) pluginDirs(includeAutoInstalled bool) []string { // When searching the following directories, earlier entries get precedence // if the same plugin version is found twice, but newer versions will @@ -97,7 +97,9 @@ func (m *Meta) pluginDirs() []string { dirs = append(dirs, filepath.Dir(exePath)) } - dirs = append(dirs, m.pluginDir()) + if includeAutoInstalled { + dirs = append(dirs, m.pluginDir()) + } dirs = append(dirs, m.GlobalPluginDirs...) return dirs } @@ -105,7 +107,33 @@ func (m *Meta) pluginDirs() []string { // providerPluginSet returns the set of valid providers that were discovered in // the defined search paths. func (m *Meta) providerPluginSet() discovery.PluginMetaSet { - plugins := discovery.FindPlugins("provider", m.pluginDirs()) + plugins := discovery.FindPlugins("provider", m.pluginDirs(true)) + plugins, _ = plugins.ValidateVersions() + + for p := range plugins { + log.Printf("[DEBUG] found valid plugin: %q", p.Name) + } + + return plugins +} + +// providerPluginAutoInstalledSet returns the set of providers that exist +// within the auto-install directory. +func (m *Meta) providerPluginAutoInstalledSet() discovery.PluginMetaSet { + plugins := discovery.FindPlugins("provider", []string{m.pluginDir()}) + plugins, _ = plugins.ValidateVersions() + + for p := range plugins { + log.Printf("[DEBUG] found valid plugin: %q", p.Name) + } + + return plugins +} + +// providerPluginManuallyInstalledSet returns the set of providers that exist +// in all locations *except* the auto-install directory. +func (m *Meta) providerPluginManuallyInstalledSet() discovery.PluginMetaSet { + plugins := discovery.FindPlugins("provider", m.pluginDirs(false)) plugins, _ = plugins.ValidateVersions() for p := range plugins { @@ -141,7 +169,7 @@ func (m *Meta) missingPlugins(avail discovery.PluginMetaSet, reqd discovery.Plug } func (m *Meta) provisionerFactories() map[string]terraform.ResourceProvisionerFactory { - dirs := m.pluginDirs() + dirs := m.pluginDirs(true) plugins := discovery.FindPlugins("provisioner", dirs) plugins, _ = plugins.ValidateVersions() diff --git a/command/plugins_test.go b/command/plugins_test.go index 8f820e158..2ee466d28 100644 --- a/command/plugins_test.go +++ b/command/plugins_test.go @@ -8,29 +8,31 @@ import ( "github.com/hashicorp/terraform/plugin/discovery" ) -// mockGetProvider providers a GetProvider method for testing automatic -// provider downloads -type mockGetProvider struct { +// mockProviderInstaller is a discovery.PluginInstaller implementation that +// is a mock for discovery.ProviderInstaller. +type mockProviderInstaller struct { // A map of provider names to available versions. // The tests expect the versions to be in order from newest to oldest. Providers map[string][]string + + Dir string + PurgeUnusedCalled bool } -func (m mockGetProvider) FileName(provider, version string) string { +func (i *mockProviderInstaller) FileName(provider, version string) string { return fmt.Sprintf("terraform-provider-%s_v%s_x4", provider, version) } -// GetProvider will check the Providers map to see if it can find a suitable -// version, and put an empty file in the dst directory. -func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constraints, protoVersion uint) error { - versions := m.Providers[provider] +func (i *mockProviderInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) { + noMeta := discovery.PluginMeta{} + versions := i.Providers[provider] if len(versions) == 0 { - return fmt.Errorf("provider %q not found", provider) + return noMeta, fmt.Errorf("provider %q not found", provider) } - err := os.MkdirAll(dst, 0755) + err := os.MkdirAll(i.Dir, 0755) if err != nil { - return fmt.Errorf("error creating plugins directory: %s", err) + return noMeta, fmt.Errorf("error creating plugins directory: %s", err) } for _, v := range versions { @@ -41,16 +43,42 @@ func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constra if req.Allows(version) { // provider filename - name := m.FileName(provider, v) - path := filepath.Join(dst, name) + name := i.FileName(provider, v) + path := filepath.Join(i.Dir, name) f, err := os.Create(path) if err != nil { - return fmt.Errorf("error fetching provider: %s", err) + return noMeta, fmt.Errorf("error fetching provider: %s", err) } f.Close() - return nil + return discovery.PluginMeta{ + Name: provider, + Version: discovery.VersionStr(v), + Path: path, + }, nil } } - return fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req) + return noMeta, fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req) +} + +func (i *mockProviderInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) { + i.PurgeUnusedCalled = true + ret := make(discovery.PluginMetaSet) + ret.Add(discovery.PluginMeta{ + Name: "test", + Version: "0.0.0", + Path: "mock-test", + }) + return ret, nil +} + +type callbackPluginInstaller func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) + +func (cb callbackPluginInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) { + return cb(provider, req) +} + +func (cb callbackPluginInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) { + // does nothing + return make(discovery.PluginMetaSet), nil } diff --git a/plugin/discovery/get.go b/plugin/discovery/get.go index ee08786b9..0e82ebcb3 100644 --- a/plugin/discovery/get.go +++ b/plugin/discovery/get.go @@ -51,24 +51,37 @@ func providerURL(name, version string) string { return u } -// GetProvider fetches a provider plugin based on the version constraints, and -// copies it to the dst directory. -// -// TODO: verify checksum and signature -func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion uint) error { +// An Installer maintains a local cache of plugins by downloading plugins +// from an online repository. +type Installer interface { + Get(name string, req Constraints) (PluginMeta, error) +} + +// ProviderInstaller is an Installer implementation that knows how to +// download Terraform providers from the official HashiCorp releases service +// into a local directory. The files downloaded are compliant with the +// naming scheme expected by FindPlugins, so the target directory of a +// provider installer can be used as one of several plugin discovery sources. +type ProviderInstaller struct { + Dir string + + PluginProtocolVersion uint +} + +func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) { versions, err := listProviderVersions(provider) // TODO: return multiple errors if err != nil { - return err + return PluginMeta{}, err } if len(versions) == 0 { - return fmt.Errorf("no plugins found for provider %q", provider) + return PluginMeta{}, fmt.Errorf("no plugins found for provider %q", provider) } versions = allowedVersions(versions, req) if len(versions) == 0 { - return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req) + return PluginMeta{}, fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req) } // sort them newest to oldest @@ -78,15 +91,53 @@ func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion ui for _, v := range versions { url := providerURL(provider, v.String()) log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) - if checkPlugin(url, pluginProtocolVersion) { + if checkPlugin(url, i.PluginProtocolVersion) { log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) - return getter.Get(dst, url) + err := getter.Get(i.Dir, url) + if err != nil { + return PluginMeta{}, err + } + + // Find what we just installed + // (This is weird, because go-getter doesn't directly return + // information about what was extracted, and we just extracted + // the archive directly into a shared dir here.) + log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v) + metas := FindPlugins("provider", []string{i.Dir}) + log.Printf("all plugins found %#v", metas) + metas, _ = metas.ValidateVersions() + metas = metas.WithName(provider).WithVersion(v) + log.Printf("filtered plugins %#v", metas) + if metas.Count() == 0 { + // This should never happen. Suggests that the release archive + // contains an executable file whose name doesn't match the + // expected convention. + return PluginMeta{}, fmt.Errorf( + "failed to find installed provider %s %s; this is a bug in Terraform and should be reported", + provider, v, + ) + } + + if metas.Count() > 1 { + // This should also never happen, and suggests that a + // particular version was re-released with a different + // executable filename. We consider releases as immutable, so + // this is an error. + return PluginMeta{}, fmt.Errorf( + "multiple plugins installed for %s %s; this is a bug in Terraform and should be reported", + provider, v, + ) + } + + // By now we know we have exactly one meta, and so "Newest" will + // return that one. + return metas.Newest(), nil } log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v) } - return fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider) + return PluginMeta{}, fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider) } // Return the plugin version by making a HEAD request to the provided url diff --git a/plugin/discovery/get_test.go b/plugin/discovery/get_test.go index af41318d1..db1643a00 100644 --- a/plugin/discovery/get_test.go +++ b/plugin/discovery/get_test.go @@ -9,8 +9,8 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "regexp" - "runtime" "strings" "testing" ) @@ -38,7 +38,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) { filename := parts[3] - reg := regexp.MustCompile(`(terraform-provider-test_(\d).(\d).(\d)_([^_]+)_([^._]+)).zip`) + reg := regexp.MustCompile(`(terraform-provider-test)_(\d).(\d).(\d)_([^_]+)_([^._]+).zip`) fileParts := reg.FindStringSubmatch(filename) if len(fileParts) != 7 { @@ -50,7 +50,8 @@ func testHandler(w http.ResponseWriter, r *http.Request) { // write a dummy file z := zip.NewWriter(w) - f, err := z.Create(fileParts[1] + "_X" + fileParts[4]) + fn := fmt.Sprintf("%s_v%s.%s.%s_x%s", fileParts[1], fileParts[2], fileParts[3], fileParts[4], fileParts[4]) + f, err := z.Create(fn) if err != nil { panic(err) } @@ -107,7 +108,7 @@ func TestCheckProtocolVersions(t *testing.T) { } } -func TestGetProvider(t *testing.T) { +func TestProviderInstaller(t *testing.T) { tmpDir, err := ioutil.TempDir("", "tf-plugin") if err != nil { t.Fatal(err) @@ -116,19 +117,38 @@ func TestGetProvider(t *testing.T) { defer os.RemoveAll(tmpDir) // attempt to use an incompatible protocol version - err = GetProvider(tmpDir, "test", AllVersions, 5) + i := &ProviderInstaller{ + Dir: tmpDir, + + PluginProtocolVersion: 5, + } + _, err = i.Get("test", AllVersions) if err == nil { - t.Fatal("protocol version is incompatible") + t.Fatal("want error for incompatible version") } - err = GetProvider(tmpDir, "test", AllVersions, 3) + i = &ProviderInstaller{ + Dir: tmpDir, + + PluginProtocolVersion: 3, + } + gotMeta, err := i.Get("test", AllVersions) if err != nil { t.Fatal(err) } // we should have version 1.2.3 - fileName := fmt.Sprintf("terraform-provider-test_1.2.3_%s_%s_X3", runtime.GOOS, runtime.GOARCH) - dest := filepath.Join(tmpDir, fileName) + dest := filepath.Join(tmpDir, "terraform-provider-test_v1.2.3_x3") + + wantMeta := PluginMeta{ + Name: "test", + Version: VersionStr("1.2.3"), + Path: dest, + } + if !reflect.DeepEqual(gotMeta, wantMeta) { + t.Errorf("wrong result meta\ngot: %#v\nwant: %#v", gotMeta, wantMeta) + } + f, err := ioutil.ReadFile(dest) if err != nil { t.Fatal(err) diff --git a/plugin/discovery/meta_set.go b/plugin/discovery/meta_set.go index 6dcfc4df2..363978443 100644 --- a/plugin/discovery/meta_set.go +++ b/plugin/discovery/meta_set.go @@ -60,6 +60,24 @@ func (s PluginMetaSet) WithName(name string) PluginMetaSet { return ns } +// WithVersion returns the subset of metas that have the given version. +// +// This should be used only with the "valid" result from ValidateVersions; +// it will ignore any plugin metas that have a invalid version strings. +func (s PluginMetaSet) WithVersion(version Version) PluginMetaSet { + ns := make(PluginMetaSet) + for p := range s { + gotVersion, err := p.Version.Parse() + if err != nil { + continue + } + if gotVersion.Equal(version) { + ns.Add(p) + } + } + return ns +} + // ByName groups the metas in the set by their Names, returning a map. func (s PluginMetaSet) ByName() map[string]PluginMetaSet { ret := make(map[string]PluginMetaSet) diff --git a/plugin/discovery/version.go b/plugin/discovery/version.go index 587ebc6e9..1dffebc97 100644 --- a/plugin/discovery/version.go +++ b/plugin/discovery/version.go @@ -49,6 +49,10 @@ func (v Version) NewerThan(other Version) bool { return v.raw.GreaterThan(other.raw) } +func (v Version) Equal(other Version) bool { + return v.raw.Equal(other.raw) +} + // MinorUpgradeConstraintStr returns a ConstraintStr that would permit // minor upgrades relative to the receiving version. func (v Version) MinorUpgradeConstraintStr() ConstraintStr {