Merge pull request #15307 from hashicorp/jbardin/plugin-vendor-dir

plugin vendor directories

command/apply.go

@@ -65,6 +65,12 @@ func (c *ApplyCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Check for user-supplied plugin path
+	if c.pluginPath, err = c.loadPluginPath(); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
+		return 1
+	}
+
 	if !c.Destroy && maybeInit {
 		// We need the pwd for the getter operation below
 		pwd, err := os.Getwd()

command/command.go

@@ -15,6 +15,15 @@ var test bool = false
 // DefaultDataDir is the default directory for storing local data.
 const DefaultDataDir = ".terraform"
 
+// PluginPathFile is the name of the file in the data dir which stores the list
+// of directories supplied by the user with the `-plugin-dir` flag during init.
+const PluginPathFile = "plugin_path"
+
+// DefaultPluginVendorDir is the location in the config directory to look for
+// user-added plugin binaries. Terraform only reads from this path if it
+// exists, it is never created by terraform.
+const DefaultPluginVendorDir = "terraform.d/plugins"
+
 // DefaultStateFilename is the default filename used for the state file.
 const DefaultStateFilename = "terraform.tfstate"
 

command/init.go

@@ -22,6 +22,9 @@ import (
 type InitCommand struct {
 	Meta
 
+	// getPlugins is for the -get-plugins flag
+	getPlugins bool
+
 	// providerInstaller is used to download and install providers that
 	// aren't found locally. This uses a discovery.ProviderInstaller instance
 	// by default, but it can be overridden here as a way to mock fetching
@@ -30,26 +33,33 @@ type InitCommand struct {
 }
 
 func (c *InitCommand) Run(args []string) int {
-	var flagBackend, flagGet, flagGetPlugins, flagUpgrade bool
+	var flagBackend, flagGet, flagUpgrade bool
 	var flagConfigExtra map[string]interface{}
+	var flagPluginPath FlagStringSlice
 
 	args = c.Meta.process(args, false)
 	cmdFlags := c.flagSet("init")
 	cmdFlags.BoolVar(&flagBackend, "backend", true, "")
 	cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "")
 	cmdFlags.BoolVar(&flagGet, "get", true, "")
-	cmdFlags.BoolVar(&flagGetPlugins, "get-plugins", true, "")
+	cmdFlags.BoolVar(&c.getPlugins, "get-plugins", true, "")
 	cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data")
 	cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
 	cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout")
 	cmdFlags.BoolVar(&c.reconfigure, "reconfigure", false, "reconfigure")
 	cmdFlags.BoolVar(&flagUpgrade, "upgrade", false, "")
+	cmdFlags.Var(&flagPluginPath, "plugin-dir", "plugin directory")
 
 	cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
 	if err := cmdFlags.Parse(args); err != nil {
 		return 1
 	}
 
+	if len(flagPluginPath) > 0 {
+		c.pluginPath = flagPluginPath
+		c.getPlugins = false
+	}
+
 	// set getProvider if we don't have a test version already
 	if c.providerInstaller == nil {
 		c.providerInstaller = &discovery.ProviderInstaller{
@@ -67,6 +77,11 @@ func (c *InitCommand) Run(args []string) int {
 		return 1
 	}
 
+	if err := c.storePluginPath(c.pluginPath); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error saving -plugin-path values: %s", err))
+		return 1
+	}
+
 	// Get our pwd. We don't always need it but always getting it is easier
 	// than the logic to determine if it is or isn't needed.
 	pwd, err := os.Getwd()
@@ -132,7 +147,7 @@ func (c *InitCommand) Run(args []string) int {
 
 		// If we're requesting backend configuration or looking for required
 		// plugins, load the backend
-		if flagBackend || flagGetPlugins {
+		if flagBackend {
 			header = true
 
 			// Only output that we're initializing a backend if we have
@@ -156,8 +171,7 @@ func (c *InitCommand) Run(args []string) int {
 		}
 	}
 
-	// Now that we have loaded all modules, check the module tree for missing providers
+	// Now that we have loaded all modules, check the module tree for missing providers.
-	if flagGetPlugins {
 	sMgr, err := back.State(c.Workspace())
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf(
@@ -181,7 +195,6 @@ func (c *InitCommand) Run(args []string) int {
 		log.Printf("[ERROR] %s", err)
 		return 1
 	}
-	}
 
 	// If we outputted information, then we need to output a newline
 	// so that our success message is nicely spaced out from prior text.
@@ -220,6 +233,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 	missing := c.missingPlugins(available, requirements)
 
 	var errs error
+	if c.getPlugins {
 		for provider, reqd := range missing {
 			c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider))
 			_, err := c.providerInstaller.Get(provider, reqd.Versions)
@@ -233,6 +247,14 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 		if errs != nil {
 			return errs
 		}
+	} else if len(missing) > 0 {
+		// we have missing providers, but aren't going to try and download them
+		for provider, reqd := range missing {
+			c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))
+			errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider))
+		}
+		return errs
+	}
 
 	// With all the providers downloaded, we'll generate our lock file
 	// that ensures the provider binaries remain unchanged until we init
@@ -351,6 +373,11 @@ Options:
 
   -no-color            If specified, output won't contain any color.
 
+  -plugin-dir          Directory containing plugin binaries. This overrides all
+                       default search paths for plugins, and prevents the 
+                       automatic installation of plugins. This flag can be used
+                       multiple times.
+
   -reconfigure         Reconfigure the backend, ignoring any saved configuration.
 
   -upgrade=false       If installing modules (-get) or plugins (-get-plugins),

command/init_test.go

@@ -494,6 +494,62 @@ func TestInit_getProvider(t *testing.T) {
 	}
 }
 
+// make sure we can locate providers in various paths
+func TestInit_findVendoredProviders(t *testing.T) {
+	// Create a temporary working directory that is empty
+	td := tempDir(t)
+
+	configDirName := "init-get-providers"
+	copy.CopyDir(testFixturePath(configDirName), filepath.Join(td, configDirName))
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	c := &InitCommand{
+		Meta:              m,
+		providerInstaller: &mockProviderInstaller{},
+	}
+
+	// make our plugin paths
+	if err := os.MkdirAll(c.pluginDir(), 0755); err != nil {
+		t.Fatal(err)
+	}
+	vendorMachineDir := filepath.Join(
+		DefaultPluginVendorDir,
+		fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH),
+	)
+	if err := os.MkdirAll(vendorMachineDir, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	// add some dummy providers
+	// the auto plugin directory
+	exactPath := filepath.Join(c.pluginDir(), "terraform-provider-exact_v1.2.3_x4")
+	if err := ioutil.WriteFile(exactPath, []byte("test bin"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	// the vendor path
+	greaterThanPath := filepath.Join(vendorMachineDir, "terraform-provider-greater_than_v2.3.4_x4")
+	if err := ioutil.WriteFile(greaterThanPath, []byte("test bin"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	// Check the current directory too
+	betweenPath := filepath.Join(".", "terraform-provider-between_v2.3.4_x4")
+	if err := ioutil.WriteFile(betweenPath, []byte("test bin"), 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	args := []string{configDirName}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
+	}
+}
+
 func TestInit_getUpgradePlugins(t *testing.T) {
 	// Create a temporary working directory that is empty
 	td := tempDir(t)
@@ -710,3 +766,101 @@ func TestInit_providerLockFile(t *testing.T) {
 		t.Errorf("wrong provider lock file contents\ngot:  %s\nwant: %s", buf, wantLockFile)
 	}
 }
+
+// Test user-supplied -plugin-dir
+func TestInit_pluginDirProviders(t *testing.T) {
+	td := tempDir(t)
+	copy.CopyDir(testFixturePath("init-get-providers"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	c := &InitCommand{
+		Meta:              m,
+		providerInstaller: &mockProviderInstaller{},
+	}
+
+	// make our vendor paths
+	pluginPath := []string{"a", "b", "c"}
+	for _, p := range pluginPath {
+		if err := os.MkdirAll(p, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// add some dummy providers in our plugin dirs
+	for i, name := range []string{
+		"terraform-provider-exact_v1.2.3_x4",
+		"terraform-provider-greater_than_v2.3.4_x4",
+		"terraform-provider-between_v2.3.4_x4",
+	} {
+
+		if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	args := []string{
+		"-plugin-dir", "a",
+		"-plugin-dir", "b",
+		"-plugin-dir", "c",
+	}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: \n%s", ui.ErrorWriter)
+	}
+}
+
+// Test user-supplied -plugin-dir doesn't allow auto-install
+func TestInit_pluginDirProvidersDoesNotGet(t *testing.T) {
+	td := tempDir(t)
+	copy.CopyDir(testFixturePath("init-get-providers"), td)
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	ui := new(cli.MockUi)
+	m := Meta{
+		testingOverrides: metaOverridesForProvider(testProvider()),
+		Ui:               ui,
+	}
+
+	c := &InitCommand{
+		Meta: m,
+		providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
+			t.Fatalf("plugin installer should not have been called for %q", provider)
+			return discovery.PluginMeta{}, nil
+		}),
+	}
+
+	// make our vendor paths
+	pluginPath := []string{"a", "b"}
+	for _, p := range pluginPath {
+		if err := os.MkdirAll(p, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// add some dummy providers in our plugin dirs
+	for i, name := range []string{
+		"terraform-provider-exact_v1.2.3_x4",
+		"terraform-provider-greater_than_v2.3.4_x4",
+	} {
+
+		if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	args := []string{
+		"-plugin-dir", "a",
+		"-plugin-dir", "b",
+	}
+	if code := c.Run(args); code == 0 {
+		// should have been an error
+		t.Fatalf("bad: \n%s", ui.OutputWriter)
+	}
+}

command/meta.go

@@ -48,6 +48,12 @@ type Meta struct {
 	// DataDir method.
 	dataDir string
 
+	// pluginPath is a user defined set of directories to look for plugins.
+	// This is set during init with the `-plugin-dir` flag, saved to a file in
+	// the data directory.
+	// This overrides all other search paths when discoverying plugins.
+	pluginPath []string
+
 	// Override certain behavior for tests within this package
 	testingOverrides *testingOverrides
 

command/plan.go

@@ -45,6 +45,12 @@ func (c *PlanCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Check for user-supplied plugin path
+	if c.pluginPath, err = c.loadPluginPath(); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
+		return 1
+	}
+
 	// Check if the path is a plan
 	plan, err := c.Plan(configPath)
 	if err != nil {

command/plugins.go

@@ -1,8 +1,11 @@
 package command
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
@@ -69,6 +72,43 @@ func (r *multiVersionProviderResolver) ResolveProviders(
 	return factories, errs
 }
 
+// store the user-supplied path for plugin discovery
+func (m *Meta) storePluginPath(pluginPath []string) error {
+	if len(pluginPath) == 0 {
+		return nil
+	}
+
+	js, err := json.MarshalIndent(pluginPath, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	// if this fails, so will WriteFile
+	os.MkdirAll(m.DataDir(), 0755)
+
+	return ioutil.WriteFile(filepath.Join(m.DataDir(), PluginPathFile), js, 0644)
+}
+
+// Load the user-defined plugin search path into Meta.pluginPath if the file
+// exists.
+func (m *Meta) loadPluginPath() ([]string, error) {
+	js, err := ioutil.ReadFile(filepath.Join(m.DataDir(), PluginPathFile))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	var pluginPath []string
+	if err := json.Unmarshal(js, &pluginPath); err != nil {
+		return nil, err
+	}
+
+	return pluginPath, nil
+}
+
 // the default location for automatically installed plugins
 func (m *Meta) pluginDir() string {
 	return filepath.Join(m.DataDir(), "plugins", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
@@ -80,6 +120,10 @@ func (m *Meta) pluginDir() string {
 // of the same plugin version are found, but newer versions always override
 // older versions where both satisfy the provider version constraints.
 func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
+	// user defined paths take precedence
+	if len(m.pluginPath) > 0 {
+		return m.pluginPath
+	}
 
 	// When searching the following directories, earlier entries get precedence
 	// if the same plugin version is found twice, but newer versions will
@@ -97,10 +141,14 @@ func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
 		dirs = append(dirs, filepath.Dir(exePath))
 	}
 
+	// add the user vendor directory
+	dirs = append(dirs, DefaultPluginVendorDir)
+
 	if includeAutoInstalled {
 		dirs = append(dirs, m.pluginDir())
 	}
 	dirs = append(dirs, m.GlobalPluginDirs...)
+
 	return dirs
 }
 

command/plugins_test.go

@@ -2,12 +2,37 @@ package command
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"reflect"
+	"testing"
 
 	"github.com/hashicorp/terraform/plugin/discovery"
 )
 
+func TestPluginPath(t *testing.T) {
+	td, err := ioutil.TempDir("", "tf")
+	defer os.RemoveAll(td)
+	defer testChdir(t, td)()
+
+	pluginPath := []string{"a", "b", "c"}
+
+	m := Meta{}
+	if err := m.storePluginPath(pluginPath); err != nil {
+		t.Fatal(err)
+	}
+
+	restoredPath, err := m.loadPluginPath()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(pluginPath, restoredPath) {
+		t.Fatalf("expected plugin path %#v, got %#v", pluginPath, restoredPath)
+	}
+}
+
 // mockProviderInstaller is a discovery.PluginInstaller implementation that
 // is a mock for discovery.ProviderInstaller.
 type mockProviderInstaller struct {

command/refresh.go

@@ -44,6 +44,12 @@ func (c *RefreshCommand) Run(args []string) int {
 		return 1
 	}
 
+	// Check for user-supplied plugin path
+	if c.pluginPath, err = c.loadPluginPath(); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
+		return 1
+	}
+
 	var conf *config.Config
 	if mod != nil {
 		conf = mod.Config()

plugin/discovery/find.go

@@ -96,6 +96,7 @@ func findPluginPaths(kind string, machineName string, dirs []string) []string {
 				continue
 			}
 
+			// FIXME: we pass in GOOS_GOARCH paths directly, so these may not be "legacy"
 			if strings.HasPrefix(fullName, prefix) {
 				// Legacy style with files directly in the base directory
 				absPath, err := filepath.Abs(filepath.Join(baseDir, fullName))