ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

plugin/discovery: Installer interface, and provider implementation 

Previously we had a "getProvider" function type used to implement plugin
fetching. Here we replace that with an interface type, initially with
just a "Get" function.

For now this just simplifies the interface by allowing the target
directory and protocol version to be members of the struct rather than
passed as arguments.

A later change will extend this interface to also include a method to
purge unused plugins, so that upgrading frequently doesn't leave behind
a trail of unused executable files.
This commit is contained in:
Martin Atkins 2017-06-12 18:22:47 -07:00
parent 5834333ea3
commit f753974bb3
8 changed files with 242 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
command/init.go
View File

@ -23,11 +23,11 @@ import (
type InitCommand struct {
Meta
// getProvider fetches providers that aren't found locally, and unpacks
// them into the dst directory.
// This uses discovery.GetProvider by default, but it provided here as a
// way to mock fetching providers for tests.
getProvider func(dst, provider string, req discovery.Constraints, protoVersion uint) error
// providerInstaller is used to download and install providers that
// aren't found locally. This uses a discovery.ProviderInstaller instance
// by default, but it can be overridden here as a way to mock fetching
// providers for tests.
providerInstaller discovery.Installer
}
func (c *InitCommand) Run(args []string) int {
@ -52,8 +52,12 @@ func (c *InitCommand) Run(args []string) int {
}
// set getProvider if we don't have a test version already
if c.getProvider == nil {
c.getProvider = discovery.GetProvider
if c.providerInstaller == nil {
c.providerInstaller = &discovery.ProviderInstaller{
Dir: c.pluginDir(),
PluginProtocolVersion: plugin.Handshake.ProtocolVersion,
}
}
// Validate the arg count
@ -176,7 +180,7 @@ func (c *InitCommand) Run(args []string) int {
"[reset][bold]Initializing provider plugins...",
))
err = c.getProviders(path, sMgr.State())
err = c.getProviders(path, sMgr.State(), flagUpgrade)
if err != nil {
// this function provides its own output
log.Printf("[ERROR] %s", err)
@ -197,7 +201,7 @@ func (c *InitCommand) Run(args []string) int {
// Load the complete module tree, and fetch any missing providers.
// This method outputs its own Ui.
func (c *InitCommand) getProviders(path string, state *terraform.State) error {
func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error {
mod, err := c.Module(path)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error getting plugins: %s", err))
@ -213,11 +217,10 @@ func (c *InitCommand) getProviders(path string, state *terraform.State) error {
requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
missing := c.missingPlugins(available, requirements)
dst := c.pluginDir()
var errs error
for provider, reqd := range missing {
c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider))
err := c.getProvider(dst, provider, reqd.Versions, plugin.Handshake.ProtocolVersion)
_, err := c.providerInstaller.Get(provider, reqd.Versions)
if err != nil {
c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))

66
command/init_test.go
View File

@ -448,7 +448,13 @@ func TestInit_getProvider(t *testing.T) {
defer os.RemoveAll(td)
defer testChdir(t, td)()
getter := &mockGetProvider{
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
installer := &mockProviderInstaller{
Providers: map[string][]string{
// looking for an exact version
"exact": []string{"1.2.3"},
@ -457,15 +463,13 @@ func TestInit_getProvider(t *testing.T) {
// config specifies
"between": []string{"3.4.5", "2.3.4", "1.2.3"},
},
Dir: m.pluginDir(),
}
ui := new(cli.MockUi)
c := &InitCommand{
Meta: Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
},
getProvider: getter.GetProvider,
Meta: m,
providerInstaller: installer,
}
args := []string{}
@ -474,15 +478,15 @@ func TestInit_getProvider(t *testing.T) {
}
// check that we got the providers for our config
exactPath := filepath.Join(c.pluginDir(), getter.FileName("exact", "1.2.3"))
exactPath := filepath.Join(c.pluginDir(), installer.FileName("exact", "1.2.3"))
if _, err := os.Stat(exactPath); os.IsNotExist(err) {
t.Fatal("provider 'exact' not downloaded")
}
greaterThanPath := filepath.Join(c.pluginDir(), getter.FileName("greater_than", "2.3.4"))
greaterThanPath := filepath.Join(c.pluginDir(), installer.FileName("greater_than", "2.3.4"))
if _, err := os.Stat(greaterThanPath); os.IsNotExist(err) {
t.Fatal("provider 'greater_than' not downloaded")
}
betweenPath := filepath.Join(c.pluginDir(), getter.FileName("between", "2.3.4"))
betweenPath := filepath.Join(c.pluginDir(), installer.FileName("between", "2.3.4"))
if _, err := os.Stat(betweenPath); os.IsNotExist(err) {
t.Fatal("provider 'between' not downloaded")
}
@ -495,7 +499,13 @@ func TestInit_getProviderMissing(t *testing.T) {
defer os.RemoveAll(td)
defer testChdir(t, td)()
getter := &mockGetProvider{
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
installer := &mockProviderInstaller{
Providers: map[string][]string{
// looking for exact version 1.2.3
"exact": []string{"1.2.4"},
@ -504,15 +514,13 @@ func TestInit_getProviderMissing(t *testing.T) {
// config specifies
"between": []string{"3.4.5", "2.3.4", "1.2.3"},
},
Dir: m.pluginDir(),
}
ui := new(cli.MockUi)
c := &InitCommand{
Meta: Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
},
getProvider: getter.GetProvider,
Meta: m,
providerInstaller: installer,
}
args := []string{}
@ -544,9 +552,9 @@ func TestInit_getProviderHaveLegacyVersion(t *testing.T) {
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
},
getProvider: func(dst, provider string, req discovery.Constraints, protoVersion uint) error {
return fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider)
},
providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
return discovery.PluginMeta{}, fmt.Errorf("EXPECTED PROVIDER ERROR %s", provider)
}),
}
args := []string{}
@ -566,19 +574,23 @@ func TestInit_providerLockFile(t *testing.T) {
defer os.RemoveAll(td)
defer testChdir(t, td)()
getter := &mockGetProvider{
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
installer := &mockProviderInstaller{
Providers: map[string][]string{
"test": []string{"1.2.3"},
},
Dir: m.pluginDir(),
}
ui := new(cli.MockUi)
c := &InitCommand{
Meta: Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
},
getProvider: getter.GetProvider,
Meta: m,
providerInstaller: installer,
}
args := []string{}

34
command/plugins.go
View File

@ -79,7 +79,7 @@ func (m *Meta) pluginDir() string {
// Earlier entries in this slice get priority over later when multiple copies
// of the same plugin version are found, but newer versions always override
// older versions where both satisfy the provider version constraints.
func (m *Meta) pluginDirs() []string {
func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
// When searching the following directories, earlier entries get precedence
// if the same plugin version is found twice, but newer versions will
@ -97,7 +97,9 @@ func (m *Meta) pluginDirs() []string {
dirs = append(dirs, filepath.Dir(exePath))
}
if includeAutoInstalled {
dirs = append(dirs, m.pluginDir())
}
dirs = append(dirs, m.GlobalPluginDirs...)
return dirs
}
@ -105,7 +107,33 @@ func (m *Meta) pluginDirs() []string {
// providerPluginSet returns the set of valid providers that were discovered in
// the defined search paths.
func (m *Meta) providerPluginSet() discovery.PluginMetaSet {
plugins := discovery.FindPlugins("provider", m.pluginDirs())
plugins := discovery.FindPlugins("provider", m.pluginDirs(true))
plugins, _ = plugins.ValidateVersions()
for p := range plugins {
log.Printf("[DEBUG] found valid plugin: %q", p.Name)
}
return plugins
}
// providerPluginAutoInstalledSet returns the set of providers that exist
// within the auto-install directory.
func (m *Meta) providerPluginAutoInstalledSet() discovery.PluginMetaSet {
plugins := discovery.FindPlugins("provider", []string{m.pluginDir()})
plugins, _ = plugins.ValidateVersions()
for p := range plugins {
log.Printf("[DEBUG] found valid plugin: %q", p.Name)
}
return plugins
}
// providerPluginManuallyInstalledSet returns the set of providers that exist
// in all locations *except* the auto-install directory.
func (m *Meta) providerPluginManuallyInstalledSet() discovery.PluginMetaSet {
plugins := discovery.FindPlugins("provider", m.pluginDirs(false))
plugins, _ = plugins.ValidateVersions()
for p := range plugins {
@ -141,7 +169,7 @@ func (m *Meta) missingPlugins(avail discovery.PluginMetaSet, reqd discovery.Plug
}
func (m *Meta) provisionerFactories() map[string]terraform.ResourceProvisionerFactory {
dirs := m.pluginDirs()
dirs := m.pluginDirs(true)
plugins := discovery.FindPlugins("provisioner", dirs)
plugins, _ = plugins.ValidateVersions()

60
command/plugins_test.go
View File

@ -8,29 +8,31 @@ import (
"github.com/hashicorp/terraform/plugin/discovery"
)
// mockGetProvider providers a GetProvider method for testing automatic
// provider downloads
type mockGetProvider struct {
// mockProviderInstaller is a discovery.PluginInstaller implementation that
// is a mock for discovery.ProviderInstaller.
type mockProviderInstaller struct {
// A map of provider names to available versions.
// The tests expect the versions to be in order from newest to oldest.
Providers map[string][]string
Dir string
PurgeUnusedCalled bool
}
func (m mockGetProvider) FileName(provider, version string) string {
func (i *mockProviderInstaller) FileName(provider, version string) string {
return fmt.Sprintf("terraform-provider-%s_v%s_x4", provider, version)
}
// GetProvider will check the Providers map to see if it can find a suitable
// version, and put an empty file in the dst directory.
func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constraints, protoVersion uint) error {
versions := m.Providers[provider]
func (i *mockProviderInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
noMeta := discovery.PluginMeta{}
versions := i.Providers[provider]
if len(versions) == 0 {
return fmt.Errorf("provider %q not found", provider)
return noMeta, fmt.Errorf("provider %q not found", provider)
}
err := os.MkdirAll(dst, 0755)
err := os.MkdirAll(i.Dir, 0755)
if err != nil {
return fmt.Errorf("error creating plugins directory: %s", err)
return noMeta, fmt.Errorf("error creating plugins directory: %s", err)
}
for _, v := range versions {
@ -41,16 +43,42 @@ func (m mockGetProvider) GetProvider(dst, provider string, req discovery.Constra
if req.Allows(version) {
// provider filename
name := m.FileName(provider, v)
path := filepath.Join(dst, name)
name := i.FileName(provider, v)
path := filepath.Join(i.Dir, name)
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("error fetching provider: %s", err)
return noMeta, fmt.Errorf("error fetching provider: %s", err)
}
f.Close()
return nil
return discovery.PluginMeta{
Name: provider,
Version: discovery.VersionStr(v),
Path: path,
}, nil
}
}
return fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req)
return noMeta, fmt.Errorf("no suitable version for provider %q found with constraints %s", provider, req)
}
func (i *mockProviderInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) {
i.PurgeUnusedCalled = true
ret := make(discovery.PluginMetaSet)
ret.Add(discovery.PluginMeta{
Name: "test",
Version: "0.0.0",
Path: "mock-test",
})
return ret, nil
}
type callbackPluginInstaller func(provider string, req discovery.Constraints) (discovery.PluginMeta, error)
func (cb callbackPluginInstaller) Get(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
return cb(provider, req)
}
func (cb callbackPluginInstaller) PurgeUnused(map[string]discovery.PluginMeta) (discovery.PluginMetaSet, error) {
// does nothing
return make(discovery.PluginMetaSet), nil
}

73
plugin/discovery/get.go
View File

@ -51,24 +51,37 @@ func providerURL(name, version string) string {
return u
}
// GetProvider fetches a provider plugin based on the version constraints, and
// copies it to the dst directory.
//
// TODO: verify checksum and signature
func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion uint) error {
// An Installer maintains a local cache of plugins by downloading plugins
// from an online repository.
type Installer interface {
Get(name string, req Constraints) (PluginMeta, error)
}
// ProviderInstaller is an Installer implementation that knows how to
// download Terraform providers from the official HashiCorp releases service
// into a local directory. The files downloaded are compliant with the
// naming scheme expected by FindPlugins, so the target directory of a
// provider installer can be used as one of several plugin discovery sources.
type ProviderInstaller struct {
Dir string
PluginProtocolVersion uint
}
func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) {
versions, err := listProviderVersions(provider)
// TODO: return multiple errors
if err != nil {
return err
return PluginMeta{}, err
}
if len(versions) == 0 {
return fmt.Errorf("no plugins found for provider %q", provider)
return PluginMeta{}, fmt.Errorf("no plugins found for provider %q", provider)
}
versions = allowedVersions(versions, req)
if len(versions) == 0 {
return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
return PluginMeta{}, fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
}
// sort them newest to oldest
@ -78,15 +91,53 @@ func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion ui
for _, v := range versions {
url := providerURL(provider, v.String())
log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
if checkPlugin(url, pluginProtocolVersion) {
if checkPlugin(url, i.PluginProtocolVersion) {
log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url)
return getter.Get(dst, url)
err := getter.Get(i.Dir, url)
if err != nil {
return PluginMeta{}, err
}
// Find what we just installed
// (This is weird, because go-getter doesn't directly return
// information about what was extracted, and we just extracted
// the archive directly into a shared dir here.)
log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v)
metas := FindPlugins("provider", []string{i.Dir})
log.Printf("all plugins found %#v", metas)
metas, _ = metas.ValidateVersions()
metas = metas.WithName(provider).WithVersion(v)
log.Printf("filtered plugins %#v", metas)
if metas.Count() == 0 {
// This should never happen. Suggests that the release archive
// contains an executable file whose name doesn't match the
// expected convention.
return PluginMeta{}, fmt.Errorf(
"failed to find installed provider %s %s; this is a bug in Terraform and should be reported",
provider, v,
)
}
if metas.Count() > 1 {
// This should also never happen, and suggests that a
// particular version was re-released with a different
// executable filename. We consider releases as immutable, so
// this is an error.
return PluginMeta{}, fmt.Errorf(
"multiple plugins installed for %s %s; this is a bug in Terraform and should be reported",
provider, v,
)
}
// By now we know we have exactly one meta, and so "Newest" will
// return that one.
return metas.Newest(), nil
}
log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v)
}
return fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
return PluginMeta{}, fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
}
// Return the plugin version by making a HEAD request to the provided url

38
plugin/discovery/get_test.go
View File

@ -9,8 +9,8 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"reflect"
"regexp"
"runtime"
"strings"
"testing"
)
@ -38,7 +38,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
filename := parts[3]
reg := regexp.MustCompile(`(terraform-provider-test_(\d).(\d).(\d)_([^_]+)_([^._]+)).zip`)
reg := regexp.MustCompile(`(terraform-provider-test)_(\d).(\d).(\d)_([^_]+)_([^._]+).zip`)
fileParts := reg.FindStringSubmatch(filename)
if len(fileParts) != 7 {
@ -50,7 +50,8 @@ func testHandler(w http.ResponseWriter, r *http.Request) {
// write a dummy file
z := zip.NewWriter(w)
f, err := z.Create(fileParts[1] + "_X" + fileParts[4])
fn := fmt.Sprintf("%s_v%s.%s.%s_x%s", fileParts[1], fileParts[2], fileParts[3], fileParts[4], fileParts[4])
f, err := z.Create(fn)
if err != nil {
panic(err)
}
@ -107,7 +108,7 @@ func TestCheckProtocolVersions(t *testing.T) {
}
}
func TestGetProvider(t *testing.T) {
func TestProviderInstaller(t *testing.T) {
tmpDir, err := ioutil.TempDir("", "tf-plugin")
if err != nil {
t.Fatal(err)
@ -116,19 +117,38 @@ func TestGetProvider(t *testing.T) {
defer os.RemoveAll(tmpDir)
// attempt to use an incompatible protocol version
err = GetProvider(tmpDir, "test", AllVersions, 5)
i := &ProviderInstaller{
Dir: tmpDir,
PluginProtocolVersion: 5,
}
_, err = i.Get("test", AllVersions)
if err == nil {
t.Fatal("protocol version is incompatible")
t.Fatal("want error for incompatible version")
}
err = GetProvider(tmpDir, "test", AllVersions, 3)
i = &ProviderInstaller{
Dir: tmpDir,
PluginProtocolVersion: 3,
}
gotMeta, err := i.Get("test", AllVersions)
if err != nil {
t.Fatal(err)
}
// we should have version 1.2.3
fileName := fmt.Sprintf("terraform-provider-test_1.2.3_%s_%s_X3", runtime.GOOS, runtime.GOARCH)
dest := filepath.Join(tmpDir, fileName)
dest := filepath.Join(tmpDir, "terraform-provider-test_v1.2.3_x3")
wantMeta := PluginMeta{
Name: "test",
Version: VersionStr("1.2.3"),
Path: dest,
}
if !reflect.DeepEqual(gotMeta, wantMeta) {
t.Errorf("wrong result meta\ngot: %#v\nwant: %#v", gotMeta, wantMeta)
}
f, err := ioutil.ReadFile(dest)
if err != nil {
t.Fatal(err)

18
plugin/discovery/meta_set.go
View File

@ -60,6 +60,24 @@ func (s PluginMetaSet) WithName(name string) PluginMetaSet {
return ns
}
// WithVersion returns the subset of metas that have the given version.
//
// This should be used only with the "valid" result from ValidateVersions;
// it will ignore any plugin metas that have a invalid version strings.
func (s PluginMetaSet) WithVersion(version Version) PluginMetaSet {
ns := make(PluginMetaSet)
for p := range s {
gotVersion, err := p.Version.Parse()
if err != nil {
continue
}
if gotVersion.Equal(version) {
ns.Add(p)
}
}
return ns
}
// ByName groups the metas in the set by their Names, returning a map.
func (s PluginMetaSet) ByName() map[string]PluginMetaSet {
ret := make(map[string]PluginMetaSet)

4
plugin/discovery/version.go
View File

@ -49,6 +49,10 @@ func (v Version) NewerThan(other Version) bool {
return v.raw.GreaterThan(other.raw)
}
func (v Version) Equal(other Version) bool {
return v.raw.Equal(other.raw)
}
// MinorUpgradeConstraintStr returns a ConstraintStr that would permit
// minor upgrades relative to the receiving version.
func (v Version) MinorUpgradeConstraintStr() ConstraintStr {