plugin/discovery: Installer interface, and provider implementation

Previously we had a "getProvider" function type used to implement plugin
fetching. Here we replace that with an interface type, initially with
just a "Get" function.

For now this just simplifies the interface by allowing the target
directory and protocol version to be members of the struct rather than
passed as arguments.

A later change will extend this interface to also include a method to
purge unused plugins, so that upgrading frequently doesn't leave behind
a trail of unused executable files.

command/init.go

@@ -23,11 +23,11 @@ import (
 type InitCommand struct {
 	Meta
 
-	// getProvider fetches providers that aren't found locally, and unpacks
+	// providerInstaller is used to download and install providers that
-	// them into the dst directory.
+	// aren't found locally. This uses a discovery.ProviderInstaller instance
-	// This uses discovery.GetProvider by default, but it provided here as a
+	// by default, but it can be overridden here as a way to mock fetching
-	// way to mock fetching providers for tests.
+	// providers for tests.
-	getProvider func(dst, provider string, req discovery.Constraints, protoVersion uint) error
+	providerInstaller discovery.Installer
 }
 
 func (c *InitCommand) Run(args []string) int {
@@ -52,8 +52,12 @@ func (c *InitCommand) Run(args []string) int {
 	}
 
 	// set getProvider if we don't have a test version already
-	if c.getProvider == nil {
+	if c.providerInstaller == nil {
-		c.getProvider = discovery.GetProvider
+		c.providerInstaller = &discovery.ProviderInstaller{
+			Dir: c.pluginDir(),
+
+			PluginProtocolVersion: plugin.Handshake.ProtocolVersion,
+		}
 	}
 
 	// Validate the arg count
@@ -176,7 +180,7 @@ func (c *InitCommand) Run(args []string) int {
 			"[reset][bold]Initializing provider plugins...",
 		))
 
-		err = c.getProviders(path, sMgr.State())
+		err = c.getProviders(path, sMgr.State(), flagUpgrade)
 		if err != nil {
 			// this function provides its own output
 			log.Printf("[ERROR] %s", err)
@@ -197,7 +201,7 @@ func (c *InitCommand) Run(args []string) int {
 
 // Load the complete module tree, and fetch any missing providers.
 // This method outputs its own Ui.
-func (c *InitCommand) getProviders(path string, state *terraform.State) error {
+func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error {
 	mod, err := c.Module(path)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error getting plugins: %s", err))
@@ -213,11 +217,10 @@ func (c *InitCommand) getProviders(path string, state *terraform.State) error {
 	requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
 	missing := c.missingPlugins(available, requirements)
 
-	dst := c.pluginDir()
 	var errs error
 	for provider, reqd := range missing {
 		c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider))
-		err := c.getProvider(dst, provider, reqd.Versions, plugin.Handshake.ProtocolVersion)
+		_, err := c.providerInstaller.Get(provider, reqd.Versions)
 
 		if err != nil {
 			c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))

command/init_test.go

@@ -448,7 +448,13 @@ func TestInit_getProvider(t *testing.T) {
 	defer os.RemoveAll(td)
 	defer testChdir(t, td)()
 
-	getter := &mockGetProvider{
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	installer := &mockProviderInstaller{
 		Providers: map[string][]string{
 			// looking for an exact version
 			"exact": []string{"1.2.3"},
@@ -457,15 +463,13 @@ func TestInit_getProvider(t *testing.T) {
 			// config specifies
 			"between": []string{"3.4.5", "2.3.4", "1.2.3"},
 		},
+
+		Dir: m.pluginDir(),
 	}
 
-	ui := new(cli.MockUi)
 	c := &InitCommand{
-		Meta: Meta{
+		Meta:              m,
-			testingOverrides: metaOverridesForProvider(testProvider()),
+		providerInstaller: installer,
-			Ui:               ui,
-		},
-		getProvider: getter.GetProvider,
 	}
 
 	args := []string{}
@@ -474,15 +478,15 @@ func TestInit_getProvider(t *testing.T) {
 	}
 
 	// check that we got the providers for our config
-	exactPath := filepath.Join(c.pluginDir(), getter.FileName("exact", "1.2.3"))
+	exactPath := filepath.Join(c.pluginDir(), installer.FileName("exact", "1.2.3"))
 	if _, err := os.Stat(exactPath); os.IsNotExist(err) {
 		t.Fatal("provider 'exact' not downloaded")
 	}
-	greaterThanPath := filepath.Join(c.pluginDir(), getter.FileName("greater_than", "2.3.4"))
+	greaterThanPath := filepath.Join(c.pluginDir(), installer.FileName("greater_than", "2.3.4"))
 	if _, err := os.Stat(greaterThanPath); os.IsNotExist(err) {
 		t.Fatal("provider 'greater_than' not downloaded")
 	}
-	betweenPath := filepath.Join(c.pluginDir(), getter.FileName("between", "2.3.4"))
+	betweenPath := filepath.Join(c.pluginDir(), installer.FileName("between", "2.3.4"))
 	if _, err := os.Stat(betweenPath); os.IsNotExist(err) {
 		t.Fatal("provider 'between' not downloaded")
 	}
@@ -495,7 +499,13 @@ func TestInit_getProviderMissing(t *testing.T) {
 	defer os.RemoveAll(td)
 	defer testChdir(t, td)()
 
-	getter := &mockGetProvider{
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	installer := &mockProviderInstaller{
 		Providers: map[string][]string{
 			// looking for exact version 1.2.3
 			"exact": []string{"1.2.4"},
@@ -504,15 +514,13 @@ func TestInit_getProviderMissing(t *testing.T) {
 			// config specifies
 			"between": []string{"3.4.5", "2.3.4", "1.2.3"},
 		},
+
+		Dir: m.pluginDir(),
 	}
 
-	ui := new(cli.MockUi)
 	c := &InitCommand{
-		Meta: Meta{
+		Meta:              m,
-			testingOverrides: metaOverridesForProvider(testProvider()),
+		providerInstaller: installer,
-			Ui:               ui,
-		},
-		getProvider: getter.GetProvider,
 	}
 
 	args := []string{}
@@ -544,9 +552,9 @@ func TestInit_getProviderHaveLegacyVersion(t *testing.T) {
 			testingOverrides: metaOverridesForProvider(testProvider()),
 			Ui:               ui,
 		},
-		getProvider: func(dst, provider string, req discovery.Constraints, protoVersion uint) error {
+		providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
-			return fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider)
+			return discovery.PluginMeta{}, fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider)
-		},
+		}),
 	}
 
 	args := []string{}
@@ -566,19 +574,23 @@ func TestInit_providerLockFile(t *testing.T) {
 	defer os.RemoveAll(td)
 	defer testChdir(t, td)()
 
-	getter := &mockGetProvider{
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	installer := &mockProviderInstaller{
 		Providers: map[string][]string{
 			"test": []string{"1.2.3"},
 		},
+
+		Dir: m.pluginDir(),
 	}
 
-	ui := new(cli.MockUi)
 	c := &InitCommand{
-		Meta: Meta{
+		Meta:              m,
-			testingOverrides: metaOverridesForProvider(testProvider()),
+		providerInstaller: installer,
-			Ui:               ui,
-		},
-		getProvider: getter.GetProvider,
 	}
 
 	args := []string{}

command/plugins.go

@@ -79,7 +79,7 @@ func (m *Meta) pluginDir() string {
 // Earlier entries in this slice get priority over later when multiple copies
 // of the same plugin version are found, but newer versions always override
 // older versions where both satisfy the provider version constraints.
-func (m *Meta) pluginDirs() []string {
+func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
 
 	// When searching the following directories, earlier entries get precedence
 	// if the same plugin version is found twice, but newer versions will
@@ -97,7 +97,9 @@ func (m *Meta) pluginDirs() []string {
 		dirs = append(dirs, filepath.Dir(exePath))
 	}
 
+	if includeAutoInstalled {
 		dirs = append(dirs, m.pluginDir())
+	}
 	dirs = append(dirs, m.GlobalPluginDirs...)
 	return dirs
 }
@@ -105,7 +107,33 @@ func (m *Meta) pluginDirs() []string {
 // providerPluginSet returns the set of valid providers that were discovered in
 // the defined search paths.
 func (m *Meta) providerPluginSet() discovery.PluginMetaSet {
-	plugins := discovery.FindPlugins("provider", m.pluginDirs())
+	plugins := discovery.FindPlugins("provider", m.pluginDirs(true))
+	plugins, _ = plugins.ValidateVersions()
+
+	for p := range plugins {
+		log.Printf("[DEBUG] found valid plugin: %q", p.Name)
+	}
+
+	return plugins
+}
+
+// providerPluginAutoInstalledSet returns the set of providers that exist
+// within the auto-install directory.
+func (m *Meta) providerPluginAutoInstalledSet() discovery.PluginMetaSet {
+	plugins := discovery.FindPlugins("provider", []string{m.pluginDir()})
+	plugins, _ = plugins.ValidateVersions()
+
+	for p := range plugins {
+		log.Printf("[DEBUG] found valid plugin: %q", p.Name)
+	}
+
+	return plugins
+}
+
+// providerPluginManuallyInstalledSet returns the set of providers that exist
+// in all locations *except* the auto-install directory.
+func (m *Meta) providerPluginManuallyInstalledSet() discovery.PluginMetaSet {
+	plugins := discovery.FindPlugins("provider", m.pluginDirs(false))
 	plugins, _ = plugins.ValidateVersions()
 
 	for p := range plugins {
@@ -141,7 +169,7 @@ func (m *Meta) missingPlugins(avail discovery.PluginMetaSet, reqd discovery.Plug
 }
 
 func (m *Meta) provisionerFactories() map[string]terraform.ResourceProvisionerFactory {
-	dirs := m.pluginDirs()
+	dirs := m.pluginDirs(true)
 	plugins := discovery.FindPlugins("provisioner", dirs)
 	plugins, _ = plugins.ValidateVersions()
 

command/plugins_test.go

@@ -8,29 +8,31 @@ import (
 	"github.com/hashicorp/terraform/plugin/discovery"
 )
 
-// mockGetProvider providers a GetProvider method for testing automatic
+// mockProviderInstaller is a discovery.PluginInstaller implementation that
-// provider downloads
+// is a mock for discovery.ProviderInstaller.
-type mockGetProvider struct {
+type mockProviderInstaller struct {
 	// A map of provider names to available versions.
 	// The tests expect the versions to be in order from newest to oldest.
 	Providers map[string][]string
+
+	Dir               string
+	PurgeUnusedCalled bool
 }
 
-func (m mockGetProvider) FileName(provider, version string) string {
+func (i *mockProviderInstaller) FileName(provider, version string) string {
 	return fmt.Sprintf("terraform-provider-%s_v%s_x4", provider, version)
 }
 
-// GetProvider will check the Providers map to see if it can find a suitable
+func (i *mockProviderInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
-// version, and put an empty file in the dst directory.
+	noMeta := discovery.PluginMeta{}
-func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constraints, protoVersion uint) error {
+	versions := i.Providers[provider]
-	versions := m.Providers[provider]
 	if len(versions) == 0 {
-		return fmt.Errorf("provider %q not found", provider)
+		return noMeta, fmt.Errorf("provider %q not found", provider)
 	}
 
-	err := os.MkdirAll(dst, 0755)
+	err := os.MkdirAll(i.Dir, 0755)
 	if err != nil {
-		return fmt.Errorf("error creating plugins directory: %s", err)
+		return noMeta, fmt.Errorf("error creating plugins directory: %s", err)
 	}
 
 	for _, v := range versions {
@@ -41,16 +43,42 @@ func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constra
 
 		if req.Allows(version) {
 			// provider filename
-			name := m.FileName(provider, v)
+			name := i.FileName(provider, v)
-			path := filepath.Join(dst, name)
+			path := filepath.Join(i.Dir, name)
 			f, err := os.Create(path)
 			if err != nil {
-				return fmt.Errorf("error fetching provider: %s", err)
+				return noMeta, fmt.Errorf("error fetching provider: %s", err)
 			}
 			f.Close()
-			return nil
+			return discovery.PluginMeta{
+				Name:    provider,
+				Version: discovery.VersionStr(v),
+				Path:    path,
+			}, nil
 		}
 	}
 
-	return fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req)
+	return noMeta, fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req)
+}
+
+func (i *mockProviderInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) {
+	i.PurgeUnusedCalled = true
+	ret := make(discovery.PluginMetaSet)
+	ret.Add(discovery.PluginMeta{
+		Name:    "test",
+		Version: "0.0.0",
+		Path:    "mock-test",
+	})
+	return ret, nil
+}
+
+type callbackPluginInstaller func(provider string, req discovery.Constraints) (discovery.PluginMeta, error)
+
+func (cb callbackPluginInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
+	return cb(provider, req)
+}
+
+func (cb callbackPluginInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) {
+	// does nothing
+	return make(discovery.PluginMetaSet), nil
 }

plugin/discovery/get.go

@@ -51,24 +51,37 @@ func providerURL(name, version string) string {
 	return u
 }
 
-// GetProvider fetches a provider plugin based on the version constraints, and
+// An Installer maintains a local cache of plugins by downloading plugins
-// copies it to the dst directory.
+// from an online repository.
-//
+type Installer interface {
-// TODO: verify checksum and signature
+	Get(name string, req Constraints) (PluginMeta, error)
-func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion uint) error {
+}
+
+// ProviderInstaller is an Installer implementation that knows how to
+// download Terraform providers from the official HashiCorp releases service
+// into a local directory. The files downloaded are compliant with the
+// naming scheme expected by FindPlugins, so the target directory of a
+// provider installer can be used as one of several plugin discovery sources.
+type ProviderInstaller struct {
+	Dir string
+
+	PluginProtocolVersion uint
+}
+
+func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) {
 	versions, err := listProviderVersions(provider)
 	// TODO: return multiple errors
 	if err != nil {
-		return err
+		return PluginMeta{}, err
 	}
 
 	if len(versions) == 0 {
-		return fmt.Errorf("no plugins found for provider %q", provider)
+		return PluginMeta{}, fmt.Errorf("no plugins found for provider %q", provider)
 	}
 
 	versions = allowedVersions(versions, req)
 	if len(versions) == 0 {
-		return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
+		return PluginMeta{}, fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
 	}
 
 	// sort them newest to oldest
@@ -78,15 +91,53 @@ func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion ui
 	for _, v := range versions {
 		url := providerURL(provider, v.String())
 		log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
-		if checkPlugin(url, pluginProtocolVersion) {
+		if checkPlugin(url, i.PluginProtocolVersion) {
 			log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url)
-			return getter.Get(dst, url)
+			err := getter.Get(i.Dir, url)
+			if err != nil {
+				return PluginMeta{}, err
+			}
+
+			// Find what we just installed
+			// (This is weird, because go-getter doesn't directly return
+			//  information about what was extracted, and we just extracted
+			//  the archive directly into a shared dir here.)
+			log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v)
+			metas := FindPlugins("provider", []string{i.Dir})
+			log.Printf("all plugins found %#v", metas)
+			metas, _ = metas.ValidateVersions()
+			metas = metas.WithName(provider).WithVersion(v)
+			log.Printf("filtered plugins %#v", metas)
+			if metas.Count() == 0 {
+				// This should never happen. Suggests that the release archive
+				// contains an executable file whose name doesn't match the
+				// expected convention.
+				return PluginMeta{}, fmt.Errorf(
+					"failed to find installed provider %s %s; this is a bug in Terraform and should be reported",
+					provider, v,
+				)
+			}
+
+			if metas.Count() > 1 {
+				// This should also never happen, and suggests that a
+				// particular version was re-released with a different
+				// executable filename. We consider releases as immutable, so
+				// this is an error.
+				return PluginMeta{}, fmt.Errorf(
+					"multiple plugins installed for %s %s; this is a bug in Terraform and should be reported",
+					provider, v,
+				)
+			}
+
+			// By now we know we have exactly one meta, and so "Newest" will
+			// return that one.
+			return metas.Newest(), nil
 		}
 
 		log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v)
 	}
 
-	return fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
+	return PluginMeta{}, fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
 }
 
 // Return the plugin version by making a HEAD request to the provided url

plugin/discovery/get_test.go

@@ -9,8 +9,8 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"reflect"
 	"regexp"
-	"runtime"
 	"strings"
 	"testing"
 )
@@ -38,7 +38,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
 
 	filename := parts[3]
 
-	reg := regexp.MustCompile(`(terraform-provider-test_(\d).(\d).(\d)_([^_]+)_([^._]+)).zip`)
+	reg := regexp.MustCompile(`(terraform-provider-test)_(\d).(\d).(\d)_([^_]+)_([^._]+).zip`)
 
 	fileParts := reg.FindStringSubmatch(filename)
 	if len(fileParts) != 7 {
@@ -50,7 +50,8 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
 
 	// write a dummy file
 	z := zip.NewWriter(w)
-	f, err := z.Create(fileParts[1] + "_X" + fileParts[4])
+	fn := fmt.Sprintf("%s_v%s.%s.%s_x%s", fileParts[1], fileParts[2], fileParts[3], fileParts[4], fileParts[4])
+	f, err := z.Create(fn)
 	if err != nil {
 		panic(err)
 	}
@@ -107,7 +108,7 @@ func TestCheckProtocolVersions(t *testing.T) {
 	}
 }
 
-func TestGetProvider(t *testing.T) {
+func TestProviderInstaller(t *testing.T) {
 	tmpDir, err := ioutil.TempDir("", "tf-plugin")
 	if err != nil {
 		t.Fatal(err)
@@ -116,19 +117,38 @@ func TestGetProvider(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 
 	// attempt to use an incompatible protocol version
-	err = GetProvider(tmpDir, "test", AllVersions, 5)
+	i := &ProviderInstaller{
+		Dir: tmpDir,
+
+		PluginProtocolVersion: 5,
+	}
+	_, err = i.Get("test", AllVersions)
 	if err == nil {
-		t.Fatal("protocol version is incompatible")
+		t.Fatal("want error for incompatible version")
 	}
 
-	err = GetProvider(tmpDir, "test", AllVersions, 3)
+	i = &ProviderInstaller{
+		Dir: tmpDir,
+
+		PluginProtocolVersion: 3,
+	}
+	gotMeta, err := i.Get("test", AllVersions)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// we should have version 1.2.3
-	fileName := fmt.Sprintf("terraform-provider-test_1.2.3_%s_%s_X3", runtime.GOOS, runtime.GOARCH)
+	dest := filepath.Join(tmpDir, "terraform-provider-test_v1.2.3_x3")
-	dest := filepath.Join(tmpDir, fileName)
+
+	wantMeta := PluginMeta{
+		Name:    "test",
+		Version: VersionStr("1.2.3"),
+		Path:    dest,
+	}
+	if !reflect.DeepEqual(gotMeta, wantMeta) {
+		t.Errorf("wrong result meta\ngot:  %#v\nwant: %#v", gotMeta, wantMeta)
+	}
+
 	f, err := ioutil.ReadFile(dest)
 	if err != nil {
 		t.Fatal(err)

plugin/discovery/meta_set.go

@@ -60,6 +60,24 @@ func (s PluginMetaSet) WithName(name string) PluginMetaSet {
 	return ns
 }
 
+// WithVersion returns the subset of metas that have the given version.
+//
+// This should be used only with the "valid" result from ValidateVersions;
+// it will ignore any plugin metas that have a invalid version strings.
+func (s PluginMetaSet) WithVersion(version Version) PluginMetaSet {
+	ns := make(PluginMetaSet)
+	for p := range s {
+		gotVersion, err := p.Version.Parse()
+		if err != nil {
+			continue
+		}
+		if gotVersion.Equal(version) {
+			ns.Add(p)
+		}
+	}
+	return ns
+}
+
 // ByName groups the metas in the set by their Names, returning a map.
 func (s PluginMetaSet) ByName() map[string]PluginMetaSet {
 	ret := make(map[string]PluginMetaSet)

plugin/discovery/version.go

@@ -49,6 +49,10 @@ func (v Version) NewerThan(other Version) bool {
 	return v.raw.GreaterThan(other.raw)
 }
 
+func (v Version) Equal(other Version) bool {
+	return v.raw.Equal(other.raw)
+}
+
 // MinorUpgradeConstraintStr returns a ConstraintStr that would permit
 // minor upgrades relative to the receiving version.
 func (v Version) MinorUpgradeConstraintStr() ConstraintStr {