From 879899d434f5686e80f4a7b1e0308b3ac06c9f27 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 1 Sep 2017 16:03:45 -0700 Subject: [PATCH] plugin/discovery: plugin caching mechanism For users that have metered or slow internet connections it is annoying to have Terraform constantly re-downloading the same files when they initialize many separate directories. To help such users, here we add an opt-in mechanism to use a local directory as a read-through cache. When enabled, any plugin download will be skipped if a suitable file already exists in the cache directory. If the desired plugin isn't in the cache, it will be downloaded into the cache for use next time. This mechanism also serves to reduce total disk usage by allowing plugin files to be shared between many configurations, as long as the target system isn't Windows and supports either hardlinks or symlinks. --- plugin/discovery/get.go | 108 +++++++++++++++++- plugin/discovery/get_cache.go | 48 ++++++++ plugin/discovery/get_cache_test.go | 29 +++++ .../terraform-provider-foo_v0.0.1_x4 | 0 4 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 plugin/discovery/get_cache.go create mode 100644 plugin/discovery/get_cache_test.go create mode 100644 plugin/discovery/test-fixtures/plugin-cache/terraform-provider-foo_v0.0.1_x4 diff --git a/plugin/discovery/get.go b/plugin/discovery/get.go index 64d2b695e..152aec047 100644 --- a/plugin/discovery/get.go +++ b/plugin/discovery/get.go @@ -3,10 +3,12 @@ package discovery import ( "errors" "fmt" + "io" "io/ioutil" "log" "net/http" "os" + "path/filepath" "runtime" "strconv" "strings" @@ -48,6 +50,10 @@ type Installer interface { type ProviderInstaller struct { Dir string + // Cache is used to access and update a local cache of plugins if non-nil. + // Can be nil to disable caching. + Cache PluginCache + PluginProtocolVersion uint // OS and Arch specify the OS and architecture that should be used when @@ -101,6 +107,12 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e // sort them newest to oldest Versions(versions).Sort() + // Ensure that our installation directory exists + err = os.MkdirAll(i.Dir, os.ModePerm) + if err != nil { + return PluginMeta{}, fmt.Errorf("failed to create plugin dir %s: %s", i.Dir, err) + } + // take the first matching plugin we find for _, v := range versions { url := i.providerURL(provider, v.String()) @@ -120,8 +132,8 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) if checkPlugin(url, i.PluginProtocolVersion) { i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", provider, v.String())) - log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) - err := getter.Get(i.Dir, url) + log.Printf("[DEBUG] getting provider %q version %q", provider, v) + err := i.install(provider, v, url) if err != nil { return PluginMeta{}, err } @@ -168,6 +180,98 @@ func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, e return PluginMeta{}, ErrorNoVersionCompatible } +func (i *ProviderInstaller) install(provider string, version Version, url string) error { + if i.Cache != nil { + log.Printf("[DEBUG] looking for provider %s %s in plugin cache", provider, version) + cached := i.Cache.CachedPluginPath("provider", provider, version) + if cached == "" { + log.Printf("[DEBUG] %s %s not yet in cache, so downloading %s", provider, version, url) + err := getter.Get(i.Cache.InstallDir(), url) + if err != nil { + return err + } + // should now be in cache + cached = i.Cache.CachedPluginPath("provider", provider, version) + if cached == "" { + // should never happen if the getter is behaving properly + // and the plugins are packaged properly. + return fmt.Errorf("failed to find downloaded plugin in cache %s", i.Cache.InstallDir()) + } + } + + // Link or copy the cached binary into our install dir so the + // normal resolution machinery can find it. + filename := filepath.Base(cached) + targetPath := filepath.Join(i.Dir, filename) + + log.Printf("[DEBUG] installing %s %s to %s from local cache %s", provider, version, targetPath, cached) + + // Delete if we can. If there's nothing there already then no harm done. + // This is important because we can't create a link if there's + // already a file of the same name present. + // (any other error here we'll catch below when we try to write here) + os.Remove(targetPath) + + // We don't attempt linking on Windows because links are not + // comprehensively supported by all tools/apps in Windows and + // so we choose to be conservative to avoid creating any + // weird issues for Windows users. + linkErr := errors.New("link not supported for Windows") // placeholder error, never actually returned + if runtime.GOOS != "windows" { + // Try hard linking first. Hard links are preferable because this + // creates a self-contained directory that doesn't depend on the + // cache after install. + linkErr = os.Link(cached, targetPath) + + // If that failed, try a symlink. This _does_ depend on the cache + // after install, so the user must manage the cache more carefully + // in this case, but avoids creating redundant copies of the + // plugins on disk. + if linkErr != nil { + linkErr = os.Symlink(cached, targetPath) + } + } + + // If we still have an error then we'll try a copy as a fallback. + // In this case either the OS is Windows or the target filesystem + // can't support symlinks. + if linkErr != nil { + srcFile, err := os.Open(cached) + if err != nil { + return fmt.Errorf("failed to open cached plugin %s: %s", cached, err) + } + defer srcFile.Close() + + destFile, err := os.OpenFile(targetPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create %s: %s", targetPath, err) + } + + _, err = io.Copy(destFile, srcFile) + if err != nil { + destFile.Close() + return fmt.Errorf("failed to copy cached plugin from %s to %s: %s", cached, targetPath, err) + } + + err = destFile.Close() + if err != nil { + return fmt.Errorf("error creating %s: %s", targetPath, err) + } + } + + // One way or another, by the time we get here we should have either + // a link or a copy of the cached plugin within i.Dir, as expected. + } else { + log.Printf("[DEBUG] plugin cache is disabled, so downloading %s %s from %s", provider, version, url) + err := getter.Get(i.Dir, url) + if err != nil { + return err + } + } + + return nil +} + func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) { purge := make(PluginMetaSet) diff --git a/plugin/discovery/get_cache.go b/plugin/discovery/get_cache.go new file mode 100644 index 000000000..1a1004264 --- /dev/null +++ b/plugin/discovery/get_cache.go @@ -0,0 +1,48 @@ +package discovery + +// PluginCache is an interface implemented by objects that are able to maintain +// a cache of plugins. +type PluginCache interface { + // CachedPluginPath returns a path where the requested plugin is already + // cached, or an empty string if the requested plugin is not yet cached. + CachedPluginPath(kind string, name string, version Version) string + + // InstallDir returns the directory that new plugins should be installed into + // in order to populate the cache. This directory should be used as the + // first argument to getter.Get when downloading plugins with go-getter. + // + // After installing into this directory, use CachedPluginPath to obtain the + // path where the plugin was installed. + InstallDir() string +} + +// NewLocalPluginCache returns a PluginCache that caches plugins in a +// given local directory. +func NewLocalPluginCache(dir string) PluginCache { + return &pluginCache{ + Dir: dir, + } +} + +type pluginCache struct { + Dir string +} + +func (c *pluginCache) CachedPluginPath(kind string, name string, version Version) string { + allPlugins := FindPlugins(kind, []string{c.Dir}) + plugins := allPlugins.WithName(name).WithVersion(version) + + if plugins.Count() == 0 { + // nothing cached + return "" + } + + // There should generally be only one plugin here; if there's more than + // one match for some reason then we'll just choose one arbitrarily. + plugin := plugins.Newest() + return plugin.Path +} + +func (c *pluginCache) InstallDir() string { + return c.Dir +} diff --git a/plugin/discovery/get_cache_test.go b/plugin/discovery/get_cache_test.go new file mode 100644 index 000000000..0753dc571 --- /dev/null +++ b/plugin/discovery/get_cache_test.go @@ -0,0 +1,29 @@ +package discovery + +import ( + "testing" +) + +func TestLocalPluginCache(t *testing.T) { + cache := NewLocalPluginCache("test-fixtures/plugin-cache") + + foo1Path := cache.CachedPluginPath("provider", "foo", VersionStr("v0.0.1").MustParse()) + if foo1Path == "" { + t.Errorf("foo v0.0.1 not found; should have been found") + } + + foo2Path := cache.CachedPluginPath("provider", "foo", VersionStr("v0.0.2").MustParse()) + if foo2Path != "" { + t.Errorf("foo v0.0.2 found at %s; should not have been found", foo2Path) + } + + baz1Path := cache.CachedPluginPath("provider", "baz", VersionStr("v0.0.1").MustParse()) + if baz1Path != "" { + t.Errorf("baz v0.0.1 found at %s; should not have been found", baz1Path) + } + + baz2Path := cache.CachedPluginPath("provider", "baz", VersionStr("v0.0.2").MustParse()) + if baz1Path != "" { + t.Errorf("baz v0.0.2 found at %s; should not have been found", baz2Path) + } +} diff --git a/plugin/discovery/test-fixtures/plugin-cache/terraform-provider-foo_v0.0.1_x4 b/plugin/discovery/test-fixtures/plugin-cache/terraform-provider-foo_v0.0.1_x4 new file mode 100644 index 000000000..e69de29bb