providercache: A package to encapsulate management of provider cache dirs

Historically our logic to handle discovering and installing providers has
been spread across several different packages. This package is intended
to become the home of all logic related to what is now called "provider
cache directories", which means directories on local disk where Terraform
caches providers in a form that is ready to run.

That includes both logic related to interrogating items already in a cache
(included in this commit) and logic related to inserting new items into
the cache from upstream provider sources (to follow in later commits).

These new codepaths are focused on providers and do not include other
plugin types (provisioners and credentials helpers), because providers are
the only plugin type that is represented by a heirarchical, decentralized
namespace and the only plugin type that has an auto-installation protocol
defined. The existing codepaths will remain to support the handling of
the other plugin types that require manual installation and that use only
a flat, locally-defined namespace.

internal/providercache/cached_provider.go

@@ -0,0 +1,33 @@
+package providercache
+
+import (
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+// CachedProvider represents a provider package in a cache directory.
+type CachedProvider struct {
+	// Provider and Version together identify the specific provider version
+	// this cache entry represents.
+	Provider addrs.Provider
+	Version  getproviders.Version
+
+	// PackageDir is the local filesystem path to the root directory where
+	// the provider's distribution archive was unpacked.
+	//
+	// The path always uses slashes as path separators, even on Windows, so
+	// that the results are consistent between platforms. Windows accepts
+	// both slashes and backslashes as long as the separators are consistent
+	// within a particular path string.
+	PackageDir string
+
+	// ExecutableFile is the local filesystem path to the main plugin executable
+	// for the provider, which is always a file within the directory given
+	// in PackageDir.
+	//
+	// The path always uses slashes as path separators, even on Windows, so
+	// that the results are consistent between platforms. Windows accepts
+	// both slashes and backslashes as long as the separators are consistent
+	// within a particular path string.
+	ExecutableFile string
+}

internal/providercache/dir.go

@@ -0,0 +1,265 @@
+package providercache
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+// Dir represents a single local filesystem directory containing cached
+// provider plugin packages that can be both read from (to find providers to
+// use for operations) and written to (during provider installation).
+//
+// The contents of a cache directory follow the same naming conventions as a
+// getproviders.FilesystemMirrorSource, except that the packages are always
+// kept in the "unpacked" form (a directory containing the contents of the
+// original distribution archive) so that they are ready for direct execution.
+//
+// A Dir also pays attention only to packages for the current host platform,
+// silently ignoring any cached packages for other platforms.
+//
+// Various Dir methods return values that are technically mutable due to the
+// restrictions of the Go typesystem, but callers are not permitted to mutate
+// any part of the returned data structures.
+type Dir struct {
+	baseDir        string
+	targetPlatform getproviders.Platform
+
+	// metaCache is a cache of the metadata of relevant packages available in
+	// the cache directory last time we scanned it. This can be nil to indicate
+	// that the cache is cold. The cache will be invalidated (set back to nil)
+	// by any operation that modifies the contents of the cache directory.
+	//
+	// We intentionally don't make effort to detect modifications to the
+	// directory made by other codepaths because the contract for NewDir
+	// explicitly defines using the same directory for multiple purposes
+	// as undefined behavior.
+	metaCache map[addrs.Provider][]CachedProvider
+}
+
+// NewDir creates and returns a new Dir object that will read and write
+// provider plugins in the given filesystem directory.
+//
+// If two instances of Dir are concurrently operating on a particular base
+// directory, or if a Dir base directory is also used as a filesystem mirror
+// source directory, the behavior is undefined.
+func NewDir(baseDir string) *Dir {
+	return &Dir{
+		baseDir:        baseDir,
+		targetPlatform: getproviders.CurrentPlatform,
+	}
+}
+
+// newDirWithPlatform is a variant of NewDir that allows selecting a specific
+// target platform, rather than taking the current one where this code is
+// running.
+//
+// This is primarily intended for portable unit testing and not particularly
+// useful in "real" callers.
+func newDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
+	return &Dir{
+		baseDir:        baseDir,
+		targetPlatform: platform,
+	}
+}
+
+// AllAvailablePackages returns a description of all of the packages already
+// present in the directory. The cache entries are grouped by the provider
+// they relate to and then sorted by version precedence, with highest
+// precedence first.
+//
+// This function will return an empty result both when the directory is empty
+// and when scanning the directory produces an error.
+//
+// The caller is forbidden from modifying the returned data structure in any
+// way, even though the Go type system permits it.
+func (d *Dir) AllAvailablePackages() map[addrs.Provider][]CachedProvider {
+	if err := d.fillMetaCache(); err != nil {
+		return nil
+	}
+
+	return d.metaCache
+}
+
+// ProviderVersion returns the cache entry for the requested provider version,
+// or nil if the requested provider version isn't present in the cache.
+func (d *Dir) ProviderVersion(provider addrs.Provider, version getproviders.Version) *CachedProvider {
+	if err := d.fillMetaCache(); err != nil {
+		return nil
+	}
+
+	for _, entry := range d.metaCache[provider] {
+		// We're intentionally comparing exact version here, so if either
+		// version number contains build metadata and they don't match then
+		// this will not return true. The rule of ignoring build metadata
+		// applies only for handling version _constraints_ and for deciding
+		// version precedence.
+		if entry.Version == version {
+			return &entry
+		}
+	}
+
+	return nil
+}
+
+// ProviderLatestVersion returns the cache entry for the latest
+// version of the requested provider already available in the cache, or nil if
+// there are no versions of that provider available.
+func (d *Dir) ProviderLatestVersion(provider addrs.Provider) *CachedProvider {
+	if err := d.fillMetaCache(); err != nil {
+		return nil
+	}
+
+	entries := d.metaCache[provider]
+	if len(entries) == 0 {
+		return nil
+	}
+
+	return &entries[0]
+}
+
+func (d *Dir) fillMetaCache() error {
+	// For d.metaCache we consider nil to be different than a non-nil empty
+	// map, so we can distinguish between having scanned and got an empty
+	// result vs. not having scanned successfully at all yet.
+	if d.metaCache != nil {
+		return nil
+	}
+
+	allData, err := getproviders.SearchLocalDirectory(d.baseDir)
+	if err != nil {
+		return err
+	}
+
+	// The getproviders package just returns everything it found, but we're
+	// interested only in a subset of the results:
+	// - those that are for the current platform
+	// - those that are in the "unpacked" form, ready to execute
+	// ...so we'll filter in these ways while we're constructing our final
+	// map to save as the cache.
+	//
+	// We intentionally always make a non-nil map, even if it might ultimately
+	// be empty, because we use that to recognize that the cache is populated.
+	data := make(map[addrs.Provider][]CachedProvider)
+
+	for providerAddr, metas := range allData {
+		for _, meta := range metas {
+			if meta.TargetPlatform != d.targetPlatform {
+				continue
+			}
+			if _, ok := meta.Location.(getproviders.PackageLocalDir); !ok {
+				// PackageLocalDir indicates an unpacked provider package ready
+				// to execute.
+				continue
+			}
+
+			packageDir := filepath.Clean(string(meta.Location.(getproviders.PackageLocalDir)))
+			execFile := findProviderExecutableInLocalPackage(meta)
+			if execFile == "" {
+				// If the package doesn't contain a suitable executable then
+				// it isn't considered to be part of our cache.
+				continue
+			}
+
+			data[providerAddr] = append(data[providerAddr], CachedProvider{
+				Provider:       providerAddr,
+				Version:        meta.Version,
+				PackageDir:     filepath.ToSlash(packageDir),
+				ExecutableFile: filepath.ToSlash(execFile),
+			})
+		}
+	}
+
+	// After we've build our lists per provider, we'll also sort them by
+	// version precedence so that the newest available version is always at
+	// index zero. If there are two versions that differ only in build metadata
+	// then it's undefined but deterministic which one we will select here,
+	// because we're preserving the order returned by SearchLocalDirectory
+	// in that case..
+	for _, entries := range data {
+		sort.SliceStable(entries, func(i, j int) bool {
+			// We're using GreaterThan rather than LessThan here because we
+			// want these in _decreasing_ order of precedence.
+			return entries[i].Version.GreaterThan(entries[j].Version)
+		})
+	}
+
+	d.metaCache = data
+	return nil
+}
+
+// This is a helper function to peep into the unpacked directory associated
+// with the given package meta and find something that looks like it's intended
+// to be the executable file for the plugin.
+//
+// This is a bit messy and heuristic-y because historically Terraform used the
+// filename itself for local filesystem discovery, allowing some variance in
+// the filenames to capture extra metadata, whereas now we're using the
+// directory structure leading to the executable instead but need to remain
+// compatible with the executable names bundled into existing provider packages.
+//
+// It will return a zero-length string if it can't find a file following
+// the expected convention in the given directory.
+func findProviderExecutableInLocalPackage(meta getproviders.PackageMeta) string {
+	packageDir, ok := meta.Location.(getproviders.PackageLocalDir)
+	if !ok {
+		// This should never happen because the providercache package only
+		// uses the local unpacked directory layout. If anything else ends
+		// up in here then we'll indicate that no executable is available,
+		// because all other locations require a fetch/unpack step first.
+		return ""
+	}
+
+	infos, err := ioutil.ReadDir(string(packageDir))
+	if err != nil {
+		// If the directory itself doesn't exist or isn't readable then we
+		// can't access an executable in it.
+		return ""
+	}
+
+	// For a provider named e.g. tf.example.com/awesomecorp/happycloud, we
+	// expect an executable file whose name starts with
+	// "terraform-provider-happycloud", followed by zero or more additional
+	// characters. If there _are_ additional characters then the first one
+	// must be an underscore or a period, like in thse examples:
+	// - terraform-provider-happycloud_v1.0.0
+	// - terraform-provider-happycloud.exe
+	//
+	// We don't require the version in the filename to match because the
+	// executable's name is no longer authoritative, but packages of "official"
+	// providers may continue to use versioned executable names for backward
+	// compatibility with Terraform 0.12.
+	//
+	// We also presume that providers packaged for Windows will include the
+	// necessary .exe extension on their filenames but do not explicitly check
+	// for that. If there's a provider package for Windows that has a file
+	// without that suffix then it will be detected as an executable but then
+	// we'll presumably fail later trying to run it.
+	wantPrefix := "terraform-provider-" + meta.Provider.Type
+
+	// We'll visit all of the directory entries and take the first (in
+	// name-lexical order) that looks like a plausible provider executable
+	// name. A package with multiple files meeting these criteria is degenerate
+	// but we will tolerate it by ignoring the subsequent entries.
+	for _, info := range infos {
+		if info.IsDir() {
+			continue // A directory can never be an executable
+		}
+		name := info.Name()
+		if !strings.HasPrefix(name, wantPrefix) {
+			continue
+		}
+		remainder := name[len(wantPrefix):]
+		if len(remainder) > 0 && (remainder[0] != '_' && remainder[0] != '.') {
+			continue // subsequent characters must be delimited by _
+		}
+		return filepath.Join(string(packageDir), name)
+	}
+
+	// If we fall out here then nothing has matched.
+	return ""
+}

internal/providercache/dir_test.go

@@ -0,0 +1,169 @@
+package providercache
+
+import (
+	"testing"
+
+	"github.com/apparentlymart/go-versions/versions"
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+func TestDirReading(t *testing.T) {
+	testDir := "testdata/cachedir"
+
+	// We'll force using particular platforms for unit testing purposes,
+	// so that we'll get consistent results on all platforms.
+	windowsPlatform := getproviders.Platform{ // only null 2.0.0 is cached
+		OS:   "windows",
+		Arch: "amd64",
+	}
+	linuxPlatform := getproviders.Platform{ // various provider versions are cached
+		OS:   "linux",
+		Arch: "amd64",
+	}
+
+	nullProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost, "hashicorp", "null",
+	)
+	randomProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost, "hashicorp", "random",
+	)
+	nonExistProvider := addrs.NewProvider(
+		addrs.DefaultRegistryHost, "bloop", "nonexist",
+	)
+	legacyProvider := addrs.NewLegacyProvider("legacy")
+
+	t.Run("ProviderLatestVersion", func(t *testing.T) {
+		t.Run("exists", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			got := dir.ProviderLatestVersion(nullProvider)
+			want := &CachedProvider{
+				Provider: nullProvider,
+
+				// We want 2.0.0 rather than 2.1.0 because the 2.1.0 package is
+				// still packed and thus not considered to be a cache member.
+				Version: versions.MustParseVersion("2.0.0"),
+
+				PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64",
+				ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe",
+			}
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+		t.Run("no package for current platform", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			// random provider is only cached for linux_amd64 in our fixtures dir
+			got := dir.ProviderLatestVersion(randomProvider)
+			var want *CachedProvider
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+		t.Run("no versions available at all", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			// nonexist provider is not present in our fixtures dir at all
+			got := dir.ProviderLatestVersion(nonExistProvider)
+			var want *CachedProvider
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+	})
+
+	t.Run("ProviderVersion", func(t *testing.T) {
+		t.Run("exists", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("2.0.0"))
+			want := &CachedProvider{
+				Provider: nullProvider,
+				Version:  versions.MustParseVersion("2.0.0"),
+
+				PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64",
+				ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe",
+			}
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+		t.Run("specified version is not cached", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			// there is no v5.0.0 package in our fixtures dir
+			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("5.0.0"))
+			var want *CachedProvider
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+		t.Run("no package for current platform", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			// random provider 1.2.0 is only cached for linux_amd64 in our fixtures dir
+			got := dir.ProviderVersion(randomProvider, versions.MustParseVersion("1.2.0"))
+			var want *CachedProvider
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+		t.Run("no versions available at all", func(t *testing.T) {
+			dir := newDirWithPlatform(testDir, windowsPlatform)
+
+			// nonexist provider is not present in our fixtures dir at all
+			got := dir.ProviderVersion(nonExistProvider, versions.MustParseVersion("1.0.0"))
+			var want *CachedProvider
+
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+	})
+
+	t.Run("AllAvailablePackages", func(t *testing.T) {
+		dir := newDirWithPlatform(testDir, linuxPlatform)
+
+		got := dir.AllAvailablePackages()
+		want := map[addrs.Provider][]CachedProvider{
+			legacyProvider: {
+				{
+					Provider:       legacyProvider,
+					Version:        versions.MustParseVersion("1.0.0"),
+					PackageDir:     "testdata/cachedir/registry.terraform.io/-/legacy/1.0.0/linux_amd64",
+					ExecutableFile: "testdata/cachedir/registry.terraform.io/-/legacy/1.0.0/linux_amd64/terraform-provider-legacy",
+				},
+			},
+			nullProvider: {
+				{
+					Provider:       nullProvider,
+					Version:        versions.MustParseVersion("2.0.0"),
+					PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64",
+					ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64/terraform-provider-null",
+				},
+			},
+			randomProvider: {
+				{
+					Provider:       randomProvider,
+					Version:        versions.MustParseVersion("1.2.0"),
+					PackageDir:     "testdata/cachedir/registry.terraform.io/hashicorp/random/1.2.0/linux_amd64",
+					ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/random/1.2.0/linux_amd64/terraform-provider-random",
+				},
+			},
+		}
+
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+}

internal/providercache/doc.go

@@ -0,0 +1,10 @@
+// Package providercache contains the logic for auto-installing providers from
+// packages obtained elsewhere, and for managing the local directories that
+// serve as global or single-configuration caches of those auto-installed
+// providers.
+//
+// It builds on the lower-level provider source functionality provided by
+// the internal/getproviders package, adding the additional behaviors around
+// obtaining the discovered providers and placing them in the cache
+// directories for subsequent use.
+package providercache

internal/providercache/testdata/cachedir/registry.terraform.io/-/legacy/1.0.0/linux_amd64/terraform-provider-legacy

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/darwin_amd64/terraform-provider-null

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64/terraform-provider-null

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/invalid

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip

@@ -0,0 +1,5 @@
+This is just a placeholder file for discovery testing, not a real provider package.
+
+This file is what we'd find for mirrors using the "packed" mirror layout,
+where the mirror maintainer can just download the packages from upstream and
+have Terraform unpack them automatically when installing.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid_invalid_invalid.zip

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/providercache/testdata/cachedir/registry.terraform.io/hashicorp/random/1.2.0/linux_amd64/terraform-provider-random

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/providercache/testdata/cachedir/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt

@@ -0,0 +1,6 @@
+Provider plugin packages are allowed to include other files such as any static
+data they need to operate, or possibly source files if the provider is written
+in an interpreted programming language.
+
+This extra file is here just to make sure that extra files don't cause any
+misbehavior during local discovery.

internal/providercache/testdata/cachedir/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.