Merge pull request #15307 from hashicorp/jbardin/plugin-vendor-dir

plugin vendor directories
This commit is contained in:
James Bardin 2017-06-15 17:35:30 -04:00 committed by GitHub
commit 5d19148f47
10 changed files with 321 additions and 33 deletions

View File

@ -65,6 +65,12 @@ func (c *ApplyCommand) Run(args []string) int {
return 1
}
// Check for user-supplied plugin path
if c.pluginPath, err = c.loadPluginPath(); err != nil {
c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
return 1
}
if !c.Destroy && maybeInit {
// We need the pwd for the getter operation below
pwd, err := os.Getwd()

View File

@ -15,6 +15,15 @@ var test bool = false
// DefaultDataDir is the default directory for storing local data.
const DefaultDataDir = ".terraform"
// PluginPathFile is the name of the file in the data dir which stores the list
// of directories supplied by the user with the `-plugin-dir` flag during init.
const PluginPathFile = "plugin_path"
// DefaultPluginVendorDir is the location in the config directory to look for
// user-added plugin binaries. Terraform only reads from this path if it
// exists, it is never created by terraform.
const DefaultPluginVendorDir = "terraform.d/plugins"
// DefaultStateFilename is the default filename used for the state file.
const DefaultStateFilename = "terraform.tfstate"

View File

@ -22,6 +22,9 @@ import (
type InitCommand struct {
Meta
// getPlugins is for the -get-plugins flag
getPlugins bool
// providerInstaller is used to download and install providers that
// aren't found locally. This uses a discovery.ProviderInstaller instance
// by default, but it can be overridden here as a way to mock fetching
@ -30,26 +33,33 @@ type InitCommand struct {
}
func (c *InitCommand) Run(args []string) int {
var flagBackend, flagGet, flagGetPlugins, flagUpgrade bool
var flagBackend, flagGet, flagUpgrade bool
var flagConfigExtra map[string]interface{}
var flagPluginPath FlagStringSlice
args = c.Meta.process(args, false)
cmdFlags := c.flagSet("init")
cmdFlags.BoolVar(&flagBackend, "backend", true, "")
cmdFlags.Var((*variables.FlagAny)(&flagConfigExtra), "backend-config", "")
cmdFlags.BoolVar(&flagGet, "get", true, "")
cmdFlags.BoolVar(&flagGetPlugins, "get-plugins", true, "")
cmdFlags.BoolVar(&c.getPlugins, "get-plugins", true, "")
cmdFlags.BoolVar(&c.forceInitCopy, "force-copy", false, "suppress prompts about copying state data")
cmdFlags.BoolVar(&c.Meta.stateLock, "lock", true, "lock state")
cmdFlags.DurationVar(&c.Meta.stateLockTimeout, "lock-timeout", 0, "lock timeout")
cmdFlags.BoolVar(&c.reconfigure, "reconfigure", false, "reconfigure")
cmdFlags.BoolVar(&flagUpgrade, "upgrade", false, "")
cmdFlags.Var(&flagPluginPath, "plugin-dir", "plugin directory")
cmdFlags.Usage = func() { c.Ui.Error(c.Help()) }
if err := cmdFlags.Parse(args); err != nil {
return 1
}
if len(flagPluginPath) > 0 {
c.pluginPath = flagPluginPath
c.getPlugins = false
}
// set getProvider if we don't have a test version already
if c.providerInstaller == nil {
c.providerInstaller = &discovery.ProviderInstaller{
@ -67,6 +77,11 @@ func (c *InitCommand) Run(args []string) int {
return 1
}
if err := c.storePluginPath(c.pluginPath); err != nil {
c.Ui.Error(fmt.Sprintf("Error saving -plugin-path values: %s", err))
return 1
}
// Get our pwd. We don't always need it but always getting it is easier
// than the logic to determine if it is or isn't needed.
pwd, err := os.Getwd()
@ -132,7 +147,7 @@ func (c *InitCommand) Run(args []string) int {
// If we're requesting backend configuration or looking for required
// plugins, load the backend
if flagBackend || flagGetPlugins {
if flagBackend {
header = true
// Only output that we're initializing a backend if we have
@ -156,31 +171,29 @@ func (c *InitCommand) Run(args []string) int {
}
}
// Now that we have loaded all modules, check the module tree for missing providers
if flagGetPlugins {
sMgr, err := back.State(c.Workspace())
if err != nil {
c.Ui.Error(fmt.Sprintf(
"Error loading state: %s", err))
return 1
}
// Now that we have loaded all modules, check the module tree for missing providers.
sMgr, err := back.State(c.Workspace())
if err != nil {
c.Ui.Error(fmt.Sprintf(
"Error loading state: %s", err))
return 1
}
if err := sMgr.RefreshState(); err != nil {
c.Ui.Error(fmt.Sprintf(
"Error refreshing state: %s", err))
return 1
}
if err := sMgr.RefreshState(); err != nil {
c.Ui.Error(fmt.Sprintf(
"Error refreshing state: %s", err))
return 1
}
c.Ui.Output(c.Colorize().Color(
"[reset][bold]Initializing provider plugins...",
))
c.Ui.Output(c.Colorize().Color(
"[reset][bold]Initializing provider plugins...",
))
err = c.getProviders(path, sMgr.State(), flagUpgrade)
if err != nil {
// this function provides its own output
log.Printf("[ERROR] %s", err)
return 1
}
err = c.getProviders(path, sMgr.State(), flagUpgrade)
if err != nil {
// this function provides its own output
log.Printf("[ERROR] %s", err)
return 1
}
// If we outputted information, then we need to output a newline
@ -220,17 +233,26 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
missing := c.missingPlugins(available, requirements)
var errs error
for provider, reqd := range missing {
c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider))
_, err := c.providerInstaller.Get(provider, reqd.Versions)
if c.getPlugins {
for provider, reqd := range missing {
c.Ui.Output(fmt.Sprintf("- downloading plugin for provider %q...", provider))
_, err := c.providerInstaller.Get(provider, reqd.Versions)
if err != nil {
c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))
errs = multierror.Append(errs, err)
if err != nil {
c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))
errs = multierror.Append(errs, err)
}
}
}
if errs != nil {
if errs != nil {
return errs
}
} else if len(missing) > 0 {
// we have missing providers, but aren't going to try and download them
for provider, reqd := range missing {
c.Ui.Error(fmt.Sprintf(errProviderNotFound, err, provider, reqd.Versions))
errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider))
}
return errs
}
@ -351,6 +373,11 @@ Options:
-no-color If specified, output won't contain any color.
-plugin-dir Directory containing plugin binaries. This overrides all
default search paths for plugins, and prevents the
automatic installation of plugins. This flag can be used
multiple times.
-reconfigure Reconfigure the backend, ignoring any saved configuration.
-upgrade=false If installing modules (-get) or plugins (-get-plugins),

View File

@ -494,6 +494,62 @@ func TestInit_getProvider(t *testing.T) {
}
}
// make sure we can locate providers in various paths
func TestInit_findVendoredProviders(t *testing.T) {
// Create a temporary working directory that is empty
td := tempDir(t)
configDirName := "init-get-providers"
copy.CopyDir(testFixturePath(configDirName), filepath.Join(td, configDirName))
defer os.RemoveAll(td)
defer testChdir(t, td)()
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
c := &InitCommand{
Meta: m,
providerInstaller: &mockProviderInstaller{},
}
// make our plugin paths
if err := os.MkdirAll(c.pluginDir(), 0755); err != nil {
t.Fatal(err)
}
vendorMachineDir := filepath.Join(
DefaultPluginVendorDir,
fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH),
)
if err := os.MkdirAll(vendorMachineDir, 0755); err != nil {
t.Fatal(err)
}
// add some dummy providers
// the auto plugin directory
exactPath := filepath.Join(c.pluginDir(), "terraform-provider-exact_v1.2.3_x4")
if err := ioutil.WriteFile(exactPath, []byte("test bin"), 0755); err != nil {
t.Fatal(err)
}
// the vendor path
greaterThanPath := filepath.Join(vendorMachineDir, "terraform-provider-greater_than_v2.3.4_x4")
if err := ioutil.WriteFile(greaterThanPath, []byte("test bin"), 0755); err != nil {
t.Fatal(err)
}
// Check the current directory too
betweenPath := filepath.Join(".", "terraform-provider-between_v2.3.4_x4")
if err := ioutil.WriteFile(betweenPath, []byte("test bin"), 0755); err != nil {
t.Fatal(err)
}
args := []string{configDirName}
if code := c.Run(args); code != 0 {
t.Fatalf("bad: \n%s", ui.ErrorWriter.String())
}
}
func TestInit_getUpgradePlugins(t *testing.T) {
// Create a temporary working directory that is empty
td := tempDir(t)
@ -710,3 +766,101 @@ func TestInit_providerLockFile(t *testing.T) {
t.Errorf("wrong provider lock file contents\ngot: %s\nwant: %s", buf, wantLockFile)
}
}
// Test user-supplied -plugin-dir
func TestInit_pluginDirProviders(t *testing.T) {
td := tempDir(t)
copy.CopyDir(testFixturePath("init-get-providers"), td)
defer os.RemoveAll(td)
defer testChdir(t, td)()
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
c := &InitCommand{
Meta: m,
providerInstaller: &mockProviderInstaller{},
}
// make our vendor paths
pluginPath := []string{"a", "b", "c"}
for _, p := range pluginPath {
if err := os.MkdirAll(p, 0755); err != nil {
t.Fatal(err)
}
}
// add some dummy providers in our plugin dirs
for i, name := range []string{
"terraform-provider-exact_v1.2.3_x4",
"terraform-provider-greater_than_v2.3.4_x4",
"terraform-provider-between_v2.3.4_x4",
} {
if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil {
t.Fatal(err)
}
}
args := []string{
"-plugin-dir", "a",
"-plugin-dir", "b",
"-plugin-dir", "c",
}
if code := c.Run(args); code != 0 {
t.Fatalf("bad: \n%s", ui.ErrorWriter)
}
}
// Test user-supplied -plugin-dir doesn't allow auto-install
func TestInit_pluginDirProvidersDoesNotGet(t *testing.T) {
td := tempDir(t)
copy.CopyDir(testFixturePath("init-get-providers"), td)
defer os.RemoveAll(td)
defer testChdir(t, td)()
ui := new(cli.MockUi)
m := Meta{
testingOverrides: metaOverridesForProvider(testProvider()),
Ui: ui,
}
c := &InitCommand{
Meta: m,
providerInstaller: callbackPluginInstaller(func(provider string, req discovery.Constraints) (discovery.PluginMeta, error) {
t.Fatalf("plugin installer should not have been called for %q", provider)
return discovery.PluginMeta{}, nil
}),
}
// make our vendor paths
pluginPath := []string{"a", "b"}
for _, p := range pluginPath {
if err := os.MkdirAll(p, 0755); err != nil {
t.Fatal(err)
}
}
// add some dummy providers in our plugin dirs
for i, name := range []string{
"terraform-provider-exact_v1.2.3_x4",
"terraform-provider-greater_than_v2.3.4_x4",
} {
if err := ioutil.WriteFile(filepath.Join(pluginPath[i], name), []byte("test bin"), 0755); err != nil {
t.Fatal(err)
}
}
args := []string{
"-plugin-dir", "a",
"-plugin-dir", "b",
}
if code := c.Run(args); code == 0 {
// should have been an error
t.Fatalf("bad: \n%s", ui.OutputWriter)
}
}

View File

@ -48,6 +48,12 @@ type Meta struct {
// DataDir method.
dataDir string
// pluginPath is a user defined set of directories to look for plugins.
// This is set during init with the `-plugin-dir` flag, saved to a file in
// the data directory.
// This overrides all other search paths when discoverying plugins.
pluginPath []string
// Override certain behavior for tests within this package
testingOverrides *testingOverrides

View File

@ -45,6 +45,12 @@ func (c *PlanCommand) Run(args []string) int {
return 1
}
// Check for user-supplied plugin path
if c.pluginPath, err = c.loadPluginPath(); err != nil {
c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
return 1
}
// Check if the path is a plan
plan, err := c.Plan(configPath)
if err != nil {

View File

@ -1,8 +1,11 @@
package command
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
@ -69,6 +72,43 @@ func (r *multiVersionProviderResolver) ResolveProviders(
return factories, errs
}
// store the user-supplied path for plugin discovery
func (m *Meta) storePluginPath(pluginPath []string) error {
if len(pluginPath) == 0 {
return nil
}
js, err := json.MarshalIndent(pluginPath, "", " ")
if err != nil {
return err
}
// if this fails, so will WriteFile
os.MkdirAll(m.DataDir(), 0755)
return ioutil.WriteFile(filepath.Join(m.DataDir(), PluginPathFile), js, 0644)
}
// Load the user-defined plugin search path into Meta.pluginPath if the file
// exists.
func (m *Meta) loadPluginPath() ([]string, error) {
js, err := ioutil.ReadFile(filepath.Join(m.DataDir(), PluginPathFile))
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}
var pluginPath []string
if err := json.Unmarshal(js, &pluginPath); err != nil {
return nil, err
}
return pluginPath, nil
}
// the default location for automatically installed plugins
func (m *Meta) pluginDir() string {
return filepath.Join(m.DataDir(), "plugins", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
@ -80,6 +120,10 @@ func (m *Meta) pluginDir() string {
// of the same plugin version are found, but newer versions always override
// older versions where both satisfy the provider version constraints.
func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
// user defined paths take precedence
if len(m.pluginPath) > 0 {
return m.pluginPath
}
// When searching the following directories, earlier entries get precedence
// if the same plugin version is found twice, but newer versions will
@ -97,10 +141,14 @@ func (m *Meta) pluginDirs(includeAutoInstalled bool) []string {
dirs = append(dirs, filepath.Dir(exePath))
}
// add the user vendor directory
dirs = append(dirs, DefaultPluginVendorDir)
if includeAutoInstalled {
dirs = append(dirs, m.pluginDir())
}
dirs = append(dirs, m.GlobalPluginDirs...)
return dirs
}

View File

@ -2,12 +2,37 @@ package command
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/hashicorp/terraform/plugin/discovery"
)
func TestPluginPath(t *testing.T) {
td, err := ioutil.TempDir("", "tf")
defer os.RemoveAll(td)
defer testChdir(t, td)()
pluginPath := []string{"a", "b", "c"}
m := Meta{}
if err := m.storePluginPath(pluginPath); err != nil {
t.Fatal(err)
}
restoredPath, err := m.loadPluginPath()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(pluginPath, restoredPath) {
t.Fatalf("expected plugin path %#v, got %#v", pluginPath, restoredPath)
}
}
// mockProviderInstaller is a discovery.PluginInstaller implementation that
// is a mock for discovery.ProviderInstaller.
type mockProviderInstaller struct {

View File

@ -44,6 +44,12 @@ func (c *RefreshCommand) Run(args []string) int {
return 1
}
// Check for user-supplied plugin path
if c.pluginPath, err = c.loadPluginPath(); err != nil {
c.Ui.Error(fmt.Sprintf("Error loading plugin path: %s", err))
return 1
}
var conf *config.Config
if mod != nil {
conf = mod.Config()

View File

@ -96,6 +96,7 @@ func findPluginPaths(kind string, machineName string, dirs []string) []string {
continue
}
// FIXME: we pass in GOOS_GOARCH paths directly, so these may not be "legacy"
if strings.HasPrefix(fullName, prefix) {
// Legacy style with files directly in the base directory
absPath, err := filepath.Abs(filepath.Join(baseDir, fullName))