internal/getproviders: Allow PackageMeta to carry acceptable hashes

The "acceptable hashes" for a package is a set of hashes that the upstream
source considers to be good hashes for checking whether future installs
of the same provider version are considered to match this one.

Because the acceptable hashes are a package authentication concern and
they already need to be known (at least in part) to implement the
authenticators, here we add AcceptableHashes as an optional extra method
that an authenticator can implement.

Because these are hashes chosen by the upstream system, the caller must
make its own determination about their trustworthiness. The result of
authentication is likely to be an input to that, for example by
distrusting hashes produced by an authenticator that succeeds but doesn't
report having validated anything.

internal/getproviders/filesystem_mirror_source_test.go

@@ -140,6 +140,10 @@ func TestFilesystemMirrorSourcePackageMeta(t *testing.T) {
 		if diff := cmp.Diff(want, got); diff != "" {
 			t.Errorf("incorrect result\n%s", diff)
 		}
+
+		if gotHashes := got.AcceptableHashes(); len(gotHashes) != 0 {
+			t.Errorf("wrong acceptable hashes\ngot:  %#v\nwant: none", gotHashes)
+		}
 	})
 	t.Run("unavailable platform", func(t *testing.T) {
 		source := NewFilesystemMirrorSource("testdata/filesystem-mirror")

internal/getproviders/http_mirror_source.go

@@ -225,7 +225,7 @@ func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version,
 	// A network mirror might not provide any hashes at all, in which case
 	// the package has no source-defined authentication whatsoever.
 	if len(archiveMeta.Hashes) > 0 {
-		ret.Authentication = NewPackageHashAuthentication(archiveMeta.Hashes)
+		ret.Authentication = NewPackageHashAuthentication(target, archiveMeta.Hashes)
 	}
 
 	return ret, nil

internal/getproviders/http_mirror_source_test.go

@@ -124,11 +124,21 @@ func TestHTTPMirrorSource(t *testing.T) {
 			Location:       PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.0_tos_m68k.zip"),
 			Authentication: packageHashAuthentication{
 				RequiredHash: "h1:placeholder-hash",
+				ValidHashes:  []string{"h1:placeholder-hash", "h0:unacceptable-hash"},
+				Platform:     Platform{"tos", "m68k"},
 			},
 		}
 		if diff := cmp.Diff(want, got); diff != "" {
 			t.Errorf("wrong result\n%s", diff)
 		}
+
+		gotHashes := got.AcceptableHashes()
+		wantHashes := map[Platform][]string{
+			tosPlatform: {"h1:placeholder-hash", "h0:unacceptable-hash"},
+		}
+		if diff := cmp.Diff(wantHashes, gotHashes); diff != "" {
+			t.Errorf("wrong acceptable hashes\n%s", diff)
+		}
 	})
 	t.Run("PackageMeta for a version that exists and has no hash", func(t *testing.T) {
 		version := MustParseVersion("1.0.1")
@@ -238,7 +248,8 @@ func testHTTPMirrorSourceHandler(resp http.ResponseWriter, req *http.Request) {
 					"tos_m68k": {
 						"url": "terraform-provider-test_v1.0.0_tos_m68k.zip",
 						"hashes": [
-							"h1:placeholder-hash"
+							"h1:placeholder-hash",
+							"h0:unacceptable-hash"
 						]
 					}
 				}

internal/getproviders/mock_source.go

@@ -206,7 +206,7 @@ func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protoc
 		// knows what the future holds?)
 		Filename: fmt.Sprintf("terraform-provider-%s_%s_%s.zip", provider.Type, version.String(), target.String()),
 
-		Authentication: NewArchiveChecksumAuthentication(checksum),
+		Authentication: NewArchiveChecksumAuthentication(target, checksum),
 	}
 	return meta, close, nil
 }

internal/getproviders/package_authentication.go

@@ -5,9 +5,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
-	"io"
 	"log"
-	"os"
 	"strings"
 
 	"golang.org/x/crypto/openpgp"
@@ -47,6 +45,32 @@ func (t *PackageAuthenticationResult) String() string {
 	}[t.result]
 }
 
+// SignedByHashiCorp returns whether the package was authenticated as signed
+// by HashiCorp.
+func (t *PackageAuthenticationResult) SignedByHashiCorp() bool {
+	if t == nil {
+		return false
+	}
+	if t.result == officialProvider {
+		return true
+	}
+
+	return false
+}
+
+// SignedByAnyParty returns whether the package was authenticated as signed
+// by either HashiCorp or by a third-party.
+func (t *PackageAuthenticationResult) SignedByAnyParty() bool {
+	if t == nil {
+		return false
+	}
+	if t.result == officialProvider || t.result == partnerProvider || t.result == communityProvider {
+		return true
+	}
+
+	return false
+}
+
 // ThirdPartySigned returns whether the package was authenticated as signed by a party
 // other than HashiCorp.
 func (t *PackageAuthenticationResult) ThirdPartySigned() bool {
@@ -88,6 +112,44 @@ type PackageAuthentication interface {
 	AuthenticatePackage(localLocation PackageLocation) (*PackageAuthenticationResult, error)
 }
 
+// PackageAuthenticationHashes is an optional interface implemented by
+// PackageAuthentication implementations that are able to return a set of
+// hashes they would consider valid if a given PackageLocation referred to
+// a package that matched that hash string.
+//
+// This can be used to record a set of acceptable hashes for a particular
+// package in a lock file so that future install operations can determine
+// whether the package has changed since its initial installation.
+type PackageAuthenticationHashes interface {
+	PackageAuthentication
+
+	// AcceptableHashes returns a set of hash strings that this authenticator
+	// would accept as valid, grouped by platform. The order of the items
+	// in each of the slices is not significant, and may contain duplicates
+	// that are also not significant.
+	//
+	// This method's result should only be used to create a "lock" for a
+	// particular provider if an earlier call to AuthenticatePackage for
+	// the corresponding package succeeded. A caller might choose to apply
+	// differing levels of trust for the acceptable hashes depending on
+	// the authentication result: a "verified checksum" result only checked
+	// that the downloaded package matched what the source claimed, which
+	// could be considered to be less trustworthy than a check that includes
+	// verifying a signature from the origin registry, depending on what the
+	// hashes are going to be used for.
+	//
+	// Hashes are returned as strings with hashing scheme prefixes like "h1:"
+	// to record which hashing scheme the hash was created with.
+	// Implementations of PackageAuthenticationHashes may return multiple
+	// hashes with different schemes, which means that all of them are equally
+	// acceptable.
+	//
+	// Authenticators that don't use hashes as their authentication procedure
+	// will either not implement this interface or will have an implementation
+	// that returns an empty result.
+	AcceptableHashes() map[Platform][]string
+}
+
 type packageAuthenticationAll []PackageAuthentication
 
 // PackageAuthenticationAll combines several authentications together into a
@@ -98,6 +160,13 @@ type packageAuthenticationAll []PackageAuthentication
 //
 // The returned result is from the last authentication, so callers should
 // take care to order the authentications such that the strongest is last.
+//
+// The returned object also implements the AcceptableHashes method from
+// interface PackageAuthenticationHashes, returning the hashes from the
+// last of the given checks that indicates at least one acceptable hash,
+// or no hashes at all if none of the constituents indicate any. The result
+// may therefore be incomplete if there is more than one check that can provide
+// hashes and they disagree about which hashes are acceptable.
 func PackageAuthenticationAll(checks ...PackageAuthentication) PackageAuthentication {
 	return packageAuthenticationAll(checks)
 }
@@ -114,8 +183,28 @@ func (checks packageAuthenticationAll) AuthenticatePackage(localLocation Package
 	return authResult, nil
 }
 
+func (checks packageAuthenticationAll) AcceptableHashes() map[Platform][]string {
+	// The elements of checks are expected to be ordered so that the strongest
+	// one is later in the list, so we'll visit them in reverse order and
+	// take the first one that implements the interface and returns a non-empty
+	// result.
+	for i := len(checks) - 1; i >= 0; i-- {
+		check, ok := checks[i].(PackageAuthenticationHashes)
+		if !ok {
+			continue
+		}
+		allHashes := check.AcceptableHashes()
+		if len(allHashes) > 0 {
+			return allHashes
+		}
+	}
+	return nil
+}
+
 type packageHashAuthentication struct {
 	RequiredHash string
+	ValidHashes  []string
+	Platform     Platform
 }
 
 // NewPackageHashAuthentication returns a PackageAuthentication implementation
@@ -126,10 +215,12 @@ type packageHashAuthentication struct {
 // The PreferredHash function will select which of the given hashes is
 // considered by Terraform to be the strongest verification, and authentication
 // succeeds as long as that chosen hash matches.
-func NewPackageHashAuthentication(validHashes []string) PackageAuthentication {
+func NewPackageHashAuthentication(platform Platform, validHashes []string) PackageAuthentication {
 	requiredHash := PreferredHash(validHashes)
 	return packageHashAuthentication{
 		RequiredHash: requiredHash,
+		ValidHashes:  validHashes,
+		Platform:     platform,
 	}
 }
 
@@ -152,7 +243,14 @@ func (a packageHashAuthentication) AuthenticatePackage(localLocation PackageLoca
 	return nil, fmt.Errorf("provider package doesn't match the expected checksum %q", a.RequiredHash)
 }
 
+func (a packageHashAuthentication) AcceptableHashes() map[Platform][]string {
+	return map[Platform][]string{
+		a.Platform: a.ValidHashes,
+	}
+}
+
 type archiveHashAuthentication struct {
+	Platform      Platform
 	WantSHA256Sum [sha256.Size]byte
 }
 
@@ -169,8 +267,8 @@ type archiveHashAuthentication struct {
 // NewPackageHashAuthentication is preferable to use when possible because
 // it uses the newer hashing scheme (implemented by function Hash) that
 // can work with both packed and unpacked provider packages.
-func NewArchiveChecksumAuthentication(wantSHA256Sum [sha256.Size]byte) PackageAuthentication {
+func NewArchiveChecksumAuthentication(platform Platform, wantSHA256Sum [sha256.Size]byte) PackageAuthentication {
-	return archiveHashAuthentication{wantSHA256Sum}
+	return archiveHashAuthentication{platform, wantSHA256Sum}
 }
 
 func (a archiveHashAuthentication) AuthenticatePackage(localLocation PackageLocation) (*PackageAuthenticationResult, error) {

internal/getproviders/package_authentication_test.go

@@ -109,7 +109,7 @@ func TestPackageHashAuthentication_success(t *testing.T) {
 		"h1:qjsREM4DqEWECD43FcPqddZ9oxCG+IaMTxvWPciS05g=",
 	}
 
-	auth := NewPackageHashAuthentication(wantHashes)
+	auth := NewPackageHashAuthentication(Platform{"linux", "amd64"}, wantHashes)
 	result, err := auth.AuthenticatePackage(location)
 
 	wantResult := PackageAuthenticationResult{result: verifiedChecksum}
@@ -143,9 +143,9 @@ func TestPackageHashAuthentication_failure(t *testing.T) {
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
-			// Empty expected hash, either because we'll error before we
+			// Invalid expected hash, either because we'll error before we
 			// reach it, or we want to force a checksum mismatch.
-			auth := NewPackageHashAuthentication([]string{"h1:invalid"})
+			auth := NewPackageHashAuthentication(Platform{"linux", "amd64"}, []string{"h1:invalid"})
 			result, err := auth.AuthenticatePackage(test.location)
 
 			if result != nil {
@@ -172,7 +172,7 @@ func TestArchiveChecksumAuthentication_success(t *testing.T) {
 		0x5a, 0x79, 0x2a, 0xde, 0x97, 0x11, 0xf5, 0x01,
 	}
 
-	auth := NewArchiveChecksumAuthentication(wantSHA256Sum)
+	auth := NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, wantSHA256Sum)
 	result, err := auth.AuthenticatePackage(location)
 
 	wantResult := PackageAuthenticationResult{result: verifiedChecksum}
@@ -194,11 +194,11 @@ func TestArchiveChecksumAuthentication_failure(t *testing.T) {
 	}{
 		"missing file": {
 			PackageLocalArchive("testdata/no-package-here.zip"),
-			"open testdata/no-package-here.zip: no such file or directory",
+			"failed to compute checksum for testdata/no-package-here.zip: lstat testdata/no-package-here.zip: no such file or directory",
 		},
 		"checksum mismatch": {
 			PackageLocalArchive("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip"),
-			"archive has incorrect SHA-256 checksum 4fb39849f2e138eb16a18ba0c682635d781cb8c3b25901dd5a792ade9711f501 (expected 0000000000000000000000000000000000000000000000000000000000000000)",
+			"archive has incorrect checksum zh:4fb39849f2e138eb16a18ba0c682635d781cb8c3b25901dd5a792ade9711f501 (expected zh:0000000000000000000000000000000000000000000000000000000000000000)",
 		},
 		"invalid location": {
 			PackageLocalDir("testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64"),
@@ -210,7 +210,7 @@ func TestArchiveChecksumAuthentication_failure(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			// Zero expected checksum, either because we'll error before we
 			// reach it, or we want to force a checksum mismatch
-			auth := NewArchiveChecksumAuthentication([sha256.Size]byte{0})
+			auth := NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, [sha256.Size]byte{0})
 			result, err := auth.AuthenticatePackage(test.location)
 
 			if result != nil {

internal/getproviders/registry_client.go

@@ -275,6 +275,10 @@ func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, t
 		}
 	}
 
+	if body.OS != target.OS || body.Arch != target.Arch {
+		return PackageMeta{}, fmt.Errorf("registry response to request for %s archive has incorrect target %s", target, Platform{body.OS, body.Arch})
+	}
+
 	downloadURL, err := url.Parse(body.DownloadURL)
 	if err != nil {
 		return PackageMeta{}, fmt.Errorf("registry response includes invalid download URL: %s", err)
@@ -351,7 +355,7 @@ func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, t
 
 	ret.Authentication = PackageAuthenticationAll(
 		NewMatchingChecksumAuthentication(document, body.Filename, checksum),
-		NewArchiveChecksumAuthentication(checksum),
+		NewArchiveChecksumAuthentication(ret.TargetPlatform, checksum),
 		NewSignatureAuthentication(document, signature, keys),
 	)
 

internal/getproviders/registry_source_test.go

@@ -115,6 +115,7 @@ func TestSourcePackageMeta(t *testing.T) {
 		version    string
 		os, arch   string
 		want       PackageMeta
+		wantHashes map[Platform][]string
 		wantErr    string
 	}{
 		// These test cases are relying on behaviors of the fake provider
@@ -138,7 +139,7 @@ func TestSourcePackageMeta(t *testing.T) {
 						"happycloud_1.2.0.zip",
 						[32]byte{30: 0xf0, 31: 0x0d},
 					),
-					NewArchiveChecksumAuthentication([32]byte{30: 0xf0, 31: 0x0d}),
+					NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, [32]byte{30: 0xf0, 31: 0x0d}),
 					NewSignatureAuthentication(
 						[]byte("000000000000000000000000000000000000000000000000000000000000f00d happycloud_1.2.0.zip\n"),
 						[]byte("GPG signature"),
@@ -148,6 +149,11 @@ func TestSourcePackageMeta(t *testing.T) {
 					),
 				),
 			},
+			map[Platform][]string{
+				{"linux", "amd64"}: {
+					"zh:000000000000000000000000000000000000000000000000000000000000f00d",
+				},
+			},
 			``,
 		},
 		{
@@ -155,6 +161,7 @@ func TestSourcePackageMeta(t *testing.T) {
 			"1.2.0",
 			"nonexist", "amd64",
 			PackageMeta{},
+			nil,
 			`provider example.com/awesomesauce/happycloud 1.2.0 is not available for nonexist_amd64`,
 		},
 		{
@@ -162,6 +169,7 @@ func TestSourcePackageMeta(t *testing.T) {
 			"1.2.0",
 			"linux", "amd64",
 			PackageMeta{},
+			nil,
 			`host not.example.com does not offer a Terraform provider registry`,
 		},
 		{
@@ -169,6 +177,7 @@ func TestSourcePackageMeta(t *testing.T) {
 			"1.2.0",
 			"linux", "amd64",
 			PackageMeta{},
+			nil,
 			`host too-new.example.com does not support the provider registry protocol required by this Terraform version, but may be compatible with a different Terraform version`,
 		},
 		{
@@ -176,6 +185,7 @@ func TestSourcePackageMeta(t *testing.T) {
 			"1.2.0",
 			"linux", "amd64",
 			PackageMeta{},
+			nil,
 			`could not query provider registry for fails.example.com/awesomesauce/happycloud: the request failed after 2 attempts, please try again later: Get "http://placeholder-origin/fails-immediately/awesomesauce/happycloud/1.2.0/download/linux/amd64": EOF`,
 		},
 	}
@@ -220,6 +230,9 @@ func TestSourcePackageMeta(t *testing.T) {
 			if diff := cmp.Diff(test.want, got, cmpOpts); diff != "" {
 				t.Errorf("wrong result\n%s", diff)
 			}
+			if diff := cmp.Diff(test.wantHashes, got.AcceptableHashes()); diff != "" {
+				t.Errorf("wrong AcceptableHashes result\n%s", diff)
+			}
 		})
 	}
 

internal/getproviders/types.go

@@ -244,6 +244,29 @@ func (m PackageMeta) PackedFilePath(baseDir string) string {
 	return PackedFilePathForPackage(baseDir, m.Provider, m.Version, m.TargetPlatform)
 }
 
+// AcceptableHashes returns a set of hashes (grouped by target platform) that
+// could be recorded for comparison to future results for the same provider
+// version, to implement a "trust on first use" scheme.
+//
+// Callers of this method should typically also verify the package using the
+// object in the Authentication field, and consider how much trust to give
+// the result of this method depending on the authentication result: an
+// unauthenticated result or one that only verified a checksum could be
+// considered less trustworthy than one that checked the package against
+// a signature provided by the origin registry.
+//
+// The AcceptableHashes result is actually provided by the object in the
+// Authentication field. AcceptableHashes therefore returns an empty result
+// for a PackageMeta that has no authentication object, or has one that does
+// not make use of hashes.
+func (m PackageMeta) AcceptableHashes() map[Platform][]string {
+	auth, ok := m.Authentication.(PackageAuthenticationHashes)
+	if !ok {
+		return nil
+	}
+	return auth.AcceptableHashes()
+}
+
 // PackageLocation represents a location where a provider distribution package
 // can be obtained. A value of this type contains one of the following
 // concrete types: PackageLocalArchive, PackageLocalDir, or PackageHTTPURL.