ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

internal/getproviders: Package hashing for local filesystem packages 

We previously had this functionality available for cached packages in the
providercache package. This moves the main implementation of this over
to the getproviders package and then implements it also for PackageMeta,
allowing us to compute hashes in a consistent way across both of our
representations of a provider package.

The new methods on PackageMeta will only be effective for packages in the
local filesystem because we need direct access to the contents in order
to produce the hash. Hopefully in future the registry protocol will be
able to also provide hashes using this content-based (rather than
archive-based) algorithm and then we'll be able to make this work for
PackageMeta referring to a package obtained from a registry too, but
hashes for local packages only are still useful for some cases right now,
such as generating mirror directories in the "terraform providers mirror"
command.
This commit is contained in:
Martin Atkins 2020-04-24 15:37:42 -07:00
parent 67311f73fd
commit 9489672d54
2 changed files with 174 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
internal/getproviders/hash.go Normal file
View File

@ -0,0 +1,157 @@
package getproviders
import (
"fmt"
"path/filepath"
"strings"
"golang.org/x/mod/sumdb/dirhash"
)
// PackageHash computes a hash of the contents of the package at the given
// location, using whichever hash algorithm is the current default.
//
// Currently, this method returns version 1 hashes as produced by the
// function PackageHashV1, but this function may switch to other versions in
// later releases. Call PackageHashV1 directly if you specifically need a V1
// hash.
//
// PackageHash can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func PackageHash(loc PackageLocation) (string, error) {
return PackageHashV1(loc)
}
// PackageMatchesHash returns true if the package at the given location matches
// the given hash, or false otherwise.
//
// If it cannot read from the given location, or if the given hash is in an
// unsupported format, PackageMatchesHash returns an error.
//
// There is currently only one hash format, as implemented by HashV1. However,
// if others are introduced in future PackageMatchesHash may accept multiple
// formats, and may generate errors for any formats that become obsolete.
//
// PackageMatchesHash can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func PackageMatchesHash(loc PackageLocation, want string) (bool, error) {
switch {
case strings.HasPrefix(want, "h1"):
got, err := PackageHashV1(loc)
if err != nil {
return false, err
}
return got == want, nil
default:
return false, fmt.Errorf("unsupported hash format (this may require a newer version of Terraform)")
}
}
// PackageHashV1 computes a hash of the contents of the package at the given
// location using hash algorithm 1.
//
// The hash covers the paths to files in the directory and the contents of
// those files. It does not cover other metadata about the files, such as
// permissions.
//
// This function is named "PackageHashV1" in anticipation of other hashing
// algorithms being added in a backward-compatible way in future. The result
// from PackageHashV1 always begins with the prefix "h1:" so that callers can
// distinguish the results of potentially multiple different hash algorithms in
// future.
//
// PackageHashV1 can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func PackageHashV1(loc PackageLocation) (string, error) {
// Our HashV1 is really just the Go Modules hash version 1, which is
// sufficient for our needs and already well-used for identity of
// Go Modules distribution packages. It is also blocked from incompatible
// changes by being used in a wide array of go.sum files already.
//
// In particular, it also supports computing an equivalent hash from
// an unpacked zip file, which is not important for Terraform workflow
// today but is likely to become so in future if we adopt a top-level
// lockfile mechanism that is intended to be checked in to version control,
// rather than just a transient lock for a particular local cache directory.
// (In that case we'd need to check hashes of _packed_ packages, too.)
//
// Internally, dirhash.Hash1 produces a string containing a sequence of
// newline-separated path+filehash pairs for all of the files in the
// directory, and then finally produces a hash of that string to return.
// In both cases, the hash algorithm is SHA256.
switch loc := loc.(type) {
case PackageLocalDir:
// We'll first dereference a possible symlink at our PackageDir location,
// as would be created if this package were linked in from another cache.
packageDir, err := filepath.EvalSymlinks(string(loc))
if err != nil {
return "", err
}
return dirhash.HashDir(packageDir, "", dirhash.Hash1)
case PackageLocalArchive:
archivePath, err := filepath.EvalSymlinks(string(loc))
if err != nil {
return "", err
}
return dirhash.HashZip(archivePath, dirhash.Hash1)
default:
return "", fmt.Errorf("cannot hash package at %s", loc.String())
}
}
// Hash computes a hash of the contents of the package at the location
// associated with the reciever, using whichever hash algorithm is the current
// default.
//
// This method will change to use new hash versions as they are introduced
// in future. If you need a specific hash version, call the method for that
// version directly instead, such as HashV1.
//
// Hash can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func (m PackageMeta) Hash() (string, error) {
return PackageHash(m.Location)
}
// MatchesHash returns true if the package at the location associated with
// the receiver matches the given hash, or false otherwise.
//
// If it cannot read from the given location, or if the given hash is in an
// unsupported format, MatchesHash returns an error.
//
// MatchesHash can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func (m PackageMeta) MatchesHash(want string) (bool, error) {
return PackageMatchesHash(m.Location, want)
}
// HashV1 computes a hash of the contents of the package at the location
// associated with the receiver using hash algorithm 1.
//
// The hash covers the paths to files in the directory and the contents of
// those files. It does not cover other metadata about the files, such as
// permissions.
//
// HashV1 can be used only with the two local package location types
// PackageLocalDir and PackageLocalArchive, because it needs to access the
// contents of the indicated package in order to compute the hash. If given
// a non-local location this function will always return an error.
func (m PackageMeta) HashV1() (string, error) {
return PackageHashV1(m.Location)
}

64
internal/providercache/cached_provider.go
View File

@ -1,12 +1,6 @@
package providercache
import (
"fmt"
"path/filepath"
"strings"
"golang.org/x/mod/sumdb/dirhash"
"github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/internal/getproviders"
)
@ -38,15 +32,24 @@ type CachedProvider struct {
ExecutableFile string
}
// PackageLocation returns the package directory given in the PackageDir field
// as a getproviders.PackageLocation implementation.
//
// Because cached providers are always in the unpacked structure, the result is
// always of the concrete type getproviders.PackageLocalDir.
func (cp *CachedProvider) PackageLocation() getproviders.PackageLocalDir {
return getproviders.PackageLocalDir(cp.PackageDir)
}
// Hash computes a hash of the contents of the package directory associated
// with the receiving cached provider, using whichever hash algorithm is
// the current default.
//
// Currently, this method returns version 1 hashes as produced by the
// method HashV1, but this function may switch to other versions in later
// releases. Call HashV1 directly if you specifically need a V1 hash.
// If you need a specific version of hash rather than just whichever one is
// current default, call that version's corresponding method (e.g. HashV1)
// directly instead.
func (cp *CachedProvider) Hash() (string, error) {
return cp.HashV1()
return getproviders.PackageHash(cp.PackageLocation())
}
// MatchesHash returns true if the package on disk matches the given hash,
@ -54,20 +57,10 @@ func (cp *CachedProvider) Hash() (string, error) {
// all of the files in it, or if the hash is in an unsupported format,
// CheckHash returns an error.
//
// There is currently only one hash format, as implemented by HashV1. However,
// if others are introduced in future MatchesHash may accept multiple formats,
// and may generate errors for any formats that become obsolete.
// MatchesHash may accept hashes in a number of different formats. Over time
// the set of supported formats may grow and shrink.
func (cp *CachedProvider) MatchesHash(want string) (bool, error) {
switch {
case strings.HasPrefix(want, "h1"):
got, err := cp.HashV1()
if err != nil {
return false, err
}
return got == want, nil
default:
return false, fmt.Errorf("unsupported hash format (this may require a newer version of Terraform)")
}
return getproviders.PackageMatchesHash(cp.PackageLocation(), want)
}
// HashV1 computes a hash of the contents of the package directory associated
@ -82,28 +75,5 @@ func (cp *CachedProvider) MatchesHash(want string) (bool, error) {
// HashV1 always begins with the prefix "h1:" so that callers can distinguish
// the results of potentially multiple different hash algorithms in future.
func (cp *CachedProvider) HashV1() (string, error) {
// Our HashV1 is really just the Go Modules hash version 1, which is
// sufficient for our needs and already well-used for identity of
// Go Modules distribution packages. It is also blocked from incompatible
// changes by being used in a wide array of go.sum files already.
//
// In particular, it also supports computing an equivalent hash from
// an unpacked zip file, which is not important for Terraform workflow
// today but is likely to become so in future if we adopt a top-level
// lockfile mechanism that is intended to be checked in to version control,
// rather than just a transient lock for a particular local cache directory.
// (In that case we'd need to check hashes of _packed_ packages, too.)
// We'll first dereference a possible symlink at our PackageDir location,
// as would be created if this package were linked in from another cache.
packageDir, err := filepath.EvalSymlinks(cp.PackageDir)
if err != nil {
return "", err
}
// Internally, dirhash.Hash1 produces a string containing a sequence of
// newline-separated path+filehash pairs for all of the files in the
// directory, and then finally produces a hash of that string to return.
// In both cases, the hash algorithm is SHA256.
return dirhash.HashDir(packageDir, "", dirhash.Hash1)
return getproviders.PackageHashV1(cp.PackageLocation())
}