internal/getproviders: Initial implementation of FilesystemMirrorSource

This is a basic implementation of FilesystemMirrorSource for now aimed
only at the specific use-case of scanning the cache of provider plugins
Terraform will keep under the ".terraform" directory, as part of our
interim provider installer implementation for Terraform 0.13.

The full functionality of this will grow out in later work when we
implement explicit local filesystem mirrors, but for now the goal is to
use this just to inspect the work done by the automatic installer once
we switch it to the new provider-FQN-aware directory structure.

The various FIXME comments in this are justified by the limited intended
scope of this initial implementation, and they should be resolved by
later work to use FilesystemMirrorSource explicitly for user-specified
provider package mirrors.

internal/getproviders/filesystem_mirror_source.go

@@ -1,6 +1,13 @@
 package getproviders
 
 import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	svchost "github.com/hashicorp/terraform-svchost"
 	"github.com/hashicorp/terraform/addrs"
 )
 
@@ -8,6 +15,11 @@ import (
 // from a directory prefix in the local filesystem.
 type FilesystemMirrorSource struct {
 	baseDir string
+
+	// allPackages caches the result of scanning the baseDir for all available
+	// packages on the first call that needs package availability information,
+	// to avoid re-scanning the filesystem on subsequent operations.
+	allPackages map[addrs.Provider]PackageMetaList
 }
 
 var _ Source = (*FilesystemMirrorSource)(nil)
@@ -24,14 +36,264 @@ func NewFilesystemMirrorSource(baseDir string) *FilesystemMirrorSource {
 // directory for locally-mirrored packages for the given provider, returning
 // a list of version numbers for the providers it found.
 func (s *FilesystemMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) {
-	// TODO: Implement
+	// s.allPackages is populated if scanAllVersions succeeds
-	panic("FilesystemMirrorSource.AvailableVersions not yet implemented")
+	err := s.scanAllVersions()
+	if err != nil {
+		return nil, err
+	}
+
+	// There might be multiple packages for a given version in the filesystem,
+	// but the contract here is to return distinct versions so we'll dedupe
+	// them first, then sort them, and then return them.
+	versionsMap := make(map[Version]struct{})
+	for _, m := range s.allPackages[provider] {
+		versionsMap[m.Version] = struct{}{}
+	}
+	ret := make(VersionList, 0, len(versionsMap))
+	for v := range versionsMap {
+		ret = append(ret, v)
+	}
+	ret.Sort()
+	return ret, nil
 }
 
 // PackageMeta checks to see if the source's base directory contains a
 // local copy of the distribution package for the given provider version on
 // the given target, and returns the metadata about it if so.
 func (s *FilesystemMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
-	// TODO: Implement
+	// s.allPackages is populated if scanAllVersions succeeds
-	panic("FilesystemMirrorSource.PackageMeta not yet implemented")
+	err := s.scanAllVersions()
+	if err != nil {
+		return PackageMeta{}, err
+	}
+
+	relevantPkgs := s.allPackages[provider].FilterProviderPlatformExactVersion(provider, target, version)
+	if len(relevantPkgs) == 0 {
+		// This is the local equivalent of a "404 Not Found" when retrieving
+		// a particular version from a registry or network mirror. Because
+		// the caller should've selected a version already found by
+		// AvailableVersions, the only discriminator that should fail here
+		// is the target platform, and so our error result assumes that,
+		// causing the caller to return an error like "This provider version is
+		// not compatible with aros_riscv".
+		return PackageMeta{}, ErrPlatformNotSupported{
+			Provider: provider,
+			Version:  version,
+			Platform: target,
+		}
+	}
+
+	// It's possible that there could be multiple copies of the same package
+	// available in the filesystem, if e.g. there's both a packed and an
+	// unpacked variant. For now we assume that the decision between them
+	// is arbitrary and just take the first one in the result.
+	return relevantPkgs[0], nil
+}
+
+// AllAvailablePackages scans the directory structure under the source's base
+// directory for locally-mirrored packages for all providers, returning a map
+// of the discovered packages with the fully-qualified provider names as
+// keys.
+//
+// This is not an operation generally supported by all Source implementations,
+// but the filesystem implementation offers it because we also use the
+// filesystem mirror source directly to scan our auto-install plugin directory
+// and in other automatic discovery situations.
+func (s *FilesystemMirrorSource) AllAvailablePackages() (map[addrs.Provider]PackageMetaList, error) {
+	// s.allPackages is populated if scanAllVersions succeeds
+	err := s.scanAllVersions()
+	return s.allPackages, err
+}
+
+func (s *FilesystemMirrorSource) scanAllVersions() error {
+	if s.allPackages != nil {
+		// we're distinguishing nil-ness from emptiness here so we can
+		// recognize when we've scanned the directory without errors, even
+		// if we found nothing during the scan.
+		return nil
+	}
+	ret := make(map[addrs.Provider]PackageMetaList)
+	err := filepath.Walk(s.baseDir, func(fullPath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return fmt.Errorf("cannot search %s: %s", fullPath, err)
+		}
+
+		// There are two valid directory structures that we support here...
+		// Unpacked: registry.terraform.io/hashicorp/aws/2.0.0/linux_amd64 (a directory)
+		// Packed:   registry.terraform.io/hashicorp/aws/terraform-provider-aws_2.0.0_linux_amd64.zip (a file)
+		//
+		// Both of these give us enough information to identify the package
+		// metadata.
+		fsPath, err := filepath.Rel(s.baseDir, fullPath)
+		if err != nil {
+			// This should never happen because the filepath.Walk contract is
+			// for the paths to include the base path.
+			log.Printf("[TRACE] FilesystemMirrorSource: ignoring malformed path %q during walk: %s", fullPath, err)
+			return nil
+		}
+		relPath := filepath.ToSlash(fsPath)
+		parts := strings.Split(relPath, "/")
+
+		if len(parts) < 3 {
+			// Likely a prefix of a valid path, so we'll ignore it and visit
+			// the full valid path on a later call.
+			return nil
+		}
+
+		hostnameGiven := parts[0]
+		namespace := parts[1]
+		typeName := parts[2]
+
+		hostname, err := svchost.ForComparison(hostnameGiven)
+		if err != nil {
+			log.Printf("[WARN] local provider path %q contains invalid hostname %q; ignoring", fullPath, hostnameGiven)
+			return nil
+		}
+		var providerAddr addrs.Provider
+		if namespace == addrs.LegacyProviderNamespace {
+			if hostname != addrs.DefaultRegistryHost {
+				log.Printf("[WARN] local provider path %q indicates a legacy provider not on the default registry host; ignoring", fullPath)
+				return nil
+			}
+			providerAddr = addrs.NewLegacyProvider(typeName)
+		} else {
+			providerAddr = addrs.NewProvider(hostname, namespace, typeName)
+		}
+
+		switch len(parts) {
+		case 5: // Might be unpacked layout
+			if !info.IsDir() {
+				return nil // packed layout requires a directory
+			}
+
+			versionStr := parts[3]
+			version, err := ParseVersion(versionStr)
+			if err != nil {
+				log.Printf("[WARN] ignoring local provider path %q with invalid version %q: %s", fullPath, versionStr, err)
+				return nil
+			}
+
+			platformStr := parts[4]
+			platform, err := ParsePlatform(platformStr)
+			if err != nil {
+				log.Printf("[WARN] ignoring local provider path %q with invalid platform %q: %s", fullPath, platformStr, err)
+				return nil
+			}
+
+			log.Printf("[TRACE] FilesystemMirrorSource: found %s v%s for %s at %s", providerAddr, version, platform, fullPath)
+
+			meta := PackageMeta{
+				Provider: providerAddr,
+				Version:  version,
+
+				// FIXME: How do we populate this?
+				ProtocolVersions: nil,
+				TargetPlatform:   platform,
+
+				// Because this is already unpacked, the filename is synthetic
+				// based on the standard naming scheme.
+				Filename: fmt.Sprintf("terraform-provider-%s_%s_%s.zip", providerAddr.Type, version, platform),
+				Location: PackageLocalDir(fullPath),
+
+				// FIXME: What about the SHA256Sum field? As currently specified
+				// it's a hash of the zip file, but this thing is already
+				// unpacked and so we don't have the zip file to hash.
+			}
+			ret[providerAddr] = append(ret[providerAddr], meta)
+
+		case 4: // Might be packed layout
+			if info.IsDir() {
+				return nil // packed layout requires a file
+			}
+
+			filename := filepath.Base(fsPath)
+			// the filename components are matched case-insensitively, and
+			// the normalized form of them is in lowercase so we'll convert
+			// to lowercase for comparison here. (This normalizes only for case,
+			// because that is the primary constraint affecting compatibility
+			// between filesystem implementations on different platforms;
+			// filenames are expected to be pre-normalized and valid in other
+			// regards.)
+			normFilename := strings.ToLower(filename)
+
+			// In the packed layout, the version number and target platform
+			// are derived from the package filename, but only if the
+			// filename has the expected prefix identifying it as a package
+			// for the provider in question, and the suffix identifying it
+			// as a zip file.
+			prefix := "terraform-provider-" + providerAddr.Type + "_"
+			const suffix = ".zip"
+			if !strings.HasPrefix(normFilename, prefix) {
+				log.Printf("[WARN] ignoring file %q as possible package for %s: lacks expected prefix %q", filename, providerAddr, prefix)
+				return nil
+			}
+			if !strings.HasSuffix(normFilename, suffix) {
+				log.Printf("[WARN] ignoring file %q as possible package for %s: lacks expected suffix %q", filename, providerAddr, suffix)
+				return nil
+			}
+
+			// Extract the version and target part of the filename, which
+			// will look like "2.1.0_linux_amd64"
+			infoSlice := normFilename[len(prefix) : len(normFilename)-len(suffix)]
+			infoParts := strings.Split(infoSlice, "_")
+			if len(infoParts) < 3 {
+				log.Printf("[WARN] ignoring file %q as possible package for %s: filename does not include version number, target OS, and target architecture", filename, providerAddr)
+				return nil
+			}
+
+			versionStr := infoParts[0]
+			version, err := ParseVersion(versionStr)
+			if err != nil {
+				log.Printf("[WARN] ignoring local provider path %q with invalid version %q: %s", fullPath, versionStr, err)
+				return nil
+			}
+
+			// We'll reassemble this back into a single string just so we can
+			// easily re-use our existing parser and its normalization rules.
+			platformStr := infoParts[1] + "_" + infoParts[2]
+			platform, err := ParsePlatform(platformStr)
+			if err != nil {
+				log.Printf("[WARN] ignoring local provider path %q with invalid platform %q: %s", fullPath, platformStr, err)
+				return nil
+			}
+
+			log.Printf("[TRACE] FilesystemMirrorSource: found %s v%s for %s at %s", providerAddr, version, platform, fullPath)
+
+			meta := PackageMeta{
+				Provider: providerAddr,
+				Version:  version,
+
+				// FIXME: How do we populate this?
+				ProtocolVersions: nil,
+				TargetPlatform:   platform,
+
+				// Because this is already unpacked, the filename is synthetic
+				// based on the standard naming scheme.
+				Filename: normFilename,                  // normalized filename, because this field says what it _should_ be called, not what it _is_ called
+				Location: PackageLocalArchive(fullPath), // non-normalized here, because this is the actual physical location
+
+				// TODO: Also populate the SHA256Sum field. Skipping that
+				// for now because our initial uses of this result --
+				// scanning already-installed providers in local directories,
+				// rather than explicit filesystem mirrors -- doesn't do
+				// any hash verification anyway, and this is consistent with
+				// the FIXME in the unpacked case above even though technically
+				// we _could_ populate SHA256Sum here right now.
+			}
+			ret[providerAddr] = append(ret[providerAddr], meta)
+
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	// Sort the results to be deterministic (aside from semver build metadata)
+	// and consistent with ordering from other functions.
+	for _, l := range ret {
+		l.Sort()
+	}
+	s.allPackages = ret
+	return nil
 }

internal/getproviders/filesystem_mirror_source_test.go

@@ -0,0 +1,166 @@
+package getproviders
+
+import (
+	"testing"
+
+	"github.com/apparentlymart/go-versions/versions"
+	"github.com/google/go-cmp/cmp"
+
+	svchost "github.com/hashicorp/terraform-svchost"
+	"github.com/hashicorp/terraform/addrs"
+)
+
+func TestFilesystemMirrorSourceAllAvailablePackages(t *testing.T) {
+	source := NewFilesystemMirrorSource("testdata/filesystem-mirror")
+	got, err := source.AllAvailablePackages()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := map[addrs.Provider]PackageMetaList{
+		nullProvider: {
+			{
+				Provider:       nullProvider,
+				Version:        versions.MustParseVersion("2.0.0"),
+				TargetPlatform: Platform{"darwin", "amd64"},
+				Filename:       "terraform-provider-null_2.0.0_darwin_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/darwin_amd64"),
+			},
+			{
+				Provider:       nullProvider,
+				Version:        versions.MustParseVersion("2.0.0"),
+				TargetPlatform: Platform{"linux", "amd64"},
+				Filename:       "terraform-provider-null_2.0.0_linux_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64"),
+			},
+			{
+				Provider:       nullProvider,
+				Version:        versions.MustParseVersion("2.1.0"),
+				TargetPlatform: Platform{"linux", "amd64"},
+				Filename:       "terraform-provider-null_2.1.0_linux_amd64.zip",
+				Location:       PackageLocalArchive("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip"),
+			},
+			{
+				Provider:       nullProvider,
+				Version:        versions.MustParseVersion("2.0.0"),
+				TargetPlatform: Platform{"windows", "amd64"},
+				Filename:       "terraform-provider-null_2.0.0_windows_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64"),
+			},
+		},
+		randomProvider: {
+			{
+				Provider:       randomProvider,
+				Version:        versions.MustParseVersion("1.2.0"),
+				TargetPlatform: Platform{"linux", "amd64"},
+				Filename:       "terraform-provider-random_1.2.0_linux_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/hashicorp/random/1.2.0/linux_amd64"),
+			},
+		},
+		happycloudProvider: {
+			{
+				Provider:       happycloudProvider,
+				Version:        versions.MustParseVersion("0.1.0-alpha.2"),
+				TargetPlatform: Platform{"darwin", "amd64"},
+				Filename:       "terraform-provider-happycloud_0.1.0-alpha.2_darwin_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64"),
+			},
+		},
+		legacyProvider: {
+			{
+				Provider:       legacyProvider,
+				Version:        versions.MustParseVersion("1.0.0"),
+				TargetPlatform: Platform{"linux", "amd64"},
+				Filename:       "terraform-provider-legacy_1.0.0_linux_amd64.zip",
+				Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/-/legacy/1.0.0/linux_amd64"),
+			},
+		},
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("incorrect result\n%s", diff)
+	}
+}
+
+func TestFilesystemMirrorSourceAvailableVersions(t *testing.T) {
+	source := NewFilesystemMirrorSource("testdata/filesystem-mirror")
+	got, err := source.AvailableVersions(nullProvider)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := VersionList{
+		versions.MustParseVersion("2.0.0"),
+		versions.MustParseVersion("2.1.0"),
+	}
+
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("incorrect result\n%s", diff)
+	}
+}
+
+func TestFilesystemMirrorSourcePackageMeta(t *testing.T) {
+	t.Run("available platform", func(t *testing.T) {
+		source := NewFilesystemMirrorSource("testdata/filesystem-mirror")
+		got, err := source.PackageMeta(
+			nullProvider, versions.MustParseVersion("2.0.0"), Platform{"linux", "amd64"},
+		)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		want := PackageMeta{
+			Provider:       nullProvider,
+			Version:        versions.MustParseVersion("2.0.0"),
+			TargetPlatform: Platform{"linux", "amd64"},
+			Filename:       "terraform-provider-null_2.0.0_linux_amd64.zip",
+			Location:       PackageLocalDir("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64"),
+		}
+
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("incorrect result\n%s", diff)
+		}
+	})
+	t.Run("unavailable platform", func(t *testing.T) {
+		source := NewFilesystemMirrorSource("testdata/filesystem-mirror")
+		// We'll request a version that does exist in the fixture directory,
+		// but for a platform that isn't supported.
+		_, err := source.PackageMeta(
+			nullProvider, versions.MustParseVersion("2.0.0"), Platform{"nonexist", "nonexist"},
+		)
+
+		if err == nil {
+			t.Fatalf("succeeded; want error")
+		}
+
+		// This specific error type is important so callers can use it to
+		// generate an actionable error message e.g. by checking to see if
+		// _any_ versions of this provider support the given platform, or
+		// similar helpful hints.
+		wantErr := ErrPlatformNotSupported{
+			Provider: nullProvider,
+			Version:  versions.MustParseVersion("2.0.0"),
+			Platform: Platform{"nonexist", "nonexist"},
+		}
+		if diff := cmp.Diff(wantErr, err); diff != "" {
+			t.Errorf("incorrect error\n%s", diff)
+		}
+	})
+}
+
+var nullProvider = addrs.Provider{
+	Hostname:  svchost.Hostname("registry.terraform.io"),
+	Namespace: "hashicorp",
+	Type:      "null",
+}
+var randomProvider = addrs.Provider{
+	Hostname:  svchost.Hostname("registry.terraform.io"),
+	Namespace: "hashicorp",
+	Type:      "random",
+}
+var happycloudProvider = addrs.Provider{
+	Hostname:  svchost.Hostname("tfe.example.com"),
+	Namespace: "awesomecorp",
+	Type:      "happycloud",
+}
+var legacyProvider = addrs.NewLegacyProvider("legacy")

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/-/legacy/1.0.0/linux_amd64/terraform-provider-legacy

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/darwin_amd64/terraform-provider-null

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/linux_amd64/terraform-provider-null

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/invalid

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip

@@ -0,0 +1,5 @@
+This is just a placeholder file for discovery testing, not a real provider package.
+
+This file is what we'd find for mirrors using the "packed" mirror layout,
+where the mirror maintainer can just download the packages from upstream and
+have Terraform unpack them automatically when installing.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid.zip

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_invalid_invalid_invalid.zip

@@ -0,0 +1 @@
+This should be ignored because it doesn't follow the provider package naming scheme.

internal/getproviders/testdata/filesystem-mirror/registry.terraform.io/hashicorp/random/1.2.0/linux_amd64/terraform-provider-random

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/extra-data.txt

@@ -0,0 +1,6 @@
+Provider plugin packages are allowed to include other files such as any static
+data they need to operate, or possibly source files if the provider is written
+in an interpreted programming language.
+
+This extra file is here just to make sure that extra files don't cause any
+misbehavior during local discovery.

internal/getproviders/testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64/terraform-provider-happycloud

@@ -0,0 +1 @@
+# This is just a placeholder file for discovery testing, not a real provider plugin.

internal/getproviders/types.go

@@ -98,6 +98,12 @@ type PackageMeta struct {
 
 	Filename string
 	Location PackageLocation
+
+	// FIXME: Our current hashing scheme only works for sources that have
+	// access to the original distribution archives, so this isn't always
+	// populated. Need to figure out a different approach where we can
+	// consistently hash both from an archive file and from an extracted
+	// archive to detect inconsistencies.
 	SHA256Sum [sha256.Size]byte
 
 	// TODO: Extra metadata for signature verification