internal/providercache: First pass of the actual install process
This is not tested yet, but it's a compilable strawman implementation of the necessary sequence of events to coordinate all of the moving parts of running a provider installation operation. This will inevitably see more iteration in later commits as we complete the surrounding parts and wire it up to be used by "terraform init". So far, it's just dead code not called by any other package.
This commit is contained in:
Martin Atkins
parent
03155daf98
commit
18dd0a396d
|
|
@ -8,6 +8,8 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/apparentlymart/go-versions/versions"
|
||||
"github.com/apparentlymart/go-versions/versions/constraints"
|
||||
|
||||
"github.com/hashicorp/terraform/addrs"
|
||||
)
|
||||
|
||||
|
|
@ -23,6 +25,10 @@ type VersionList = versions.List
|
|||
// by the end-user.
|
||||
type VersionSet = versions.Set
|
||||
|
||||
// VersionConstraints represents a set of version constraints, which can
|
||||
// define the membership of a VersionSet by exclusion.
|
||||
type VersionConstraints = constraints.IntersectionSpec
|
||||
|
||||
// ParseVersion parses a "semver"-style version string into a Version value,
|
||||
// which is the version syntax we use for provider versions.
|
||||
func ParseVersion(str string) (Version, error) {
|
||||
|
|
@ -246,3 +252,56 @@ func (l PackageMetaList) FilterProviderPlatformExactVersion(provider addrs.Provi
|
|||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// VersionConstraintsString returns a UI-oriented string representation of
|
||||
// a VersionConstraints value.
|
||||
func VersionConstraintsString(spec VersionConstraints) string {
|
||||
// (we have our own function for this because the upstream versions
|
||||
// library prefers to use npm/cargo-style constraint syntax, but
|
||||
// Terraform prefers Ruby-like. Maybe we can upstream a "RubyLikeString")
|
||||
// function to do this later, but having this in here avoids blocking on
|
||||
// that and this is the sort of thing that is unlikely to need ongoing
|
||||
// maintenance because the version constraint syntax is unlikely to change.)
|
||||
|
||||
var b strings.Builder
|
||||
for i, sel := range spec {
|
||||
if i > 0 {
|
||||
b.WriteString(", ")
|
||||
}
|
||||
switch sel.Operator {
|
||||
case constraints.OpGreaterThan:
|
||||
b.WriteString("> ")
|
||||
case constraints.OpLessThan:
|
||||
b.WriteString("< ")
|
||||
case constraints.OpGreaterThanOrEqual:
|
||||
b.WriteString(">= ")
|
||||
case constraints.OpGreaterThanOrEqualPatchOnly, constraints.OpGreaterThanOrEqualMinorOnly:
|
||||
// These two differ in how the version is written, not in the symbol.
|
||||
b.WriteString("~> ")
|
||||
case constraints.OpLessThanOrEqual:
|
||||
b.WriteString("<= ")
|
||||
case constraints.OpEqual:
|
||||
b.WriteString("")
|
||||
case constraints.OpNotEqual:
|
||||
b.WriteString("!= ")
|
||||
default:
|
||||
// The above covers all of the operators we support during
|
||||
// parsing, so we should not get here.
|
||||
b.WriteString("??? ")
|
||||
}
|
||||
|
||||
if sel.Operator == constraints.OpGreaterThanOrEqualMinorOnly {
|
||||
// The minor-pessimistic syntax uses only two version components.
|
||||
fmt.Fprintf(&b, "%s.%s", sel.Boundary.Major, sel.Boundary.Minor)
|
||||
} else {
|
||||
fmt.Fprintf(&b, "%s.%s.%s", sel.Boundary.Major, sel.Boundary.Minor, sel.Boundary.Patch)
|
||||
}
|
||||
if sel.Boundary.Prerelease != "" {
|
||||
b.WriteString("-" + sel.Boundary.Prerelease)
|
||||
}
|
||||
if sel.Boundary.Metadata != "" {
|
||||
b.WriteString("+" + sel.Boundary.Metadata)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,351 @@
|
|||
package providercache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/apparentlymart/go-versions/versions"
|
||||
|
||||
"github.com/hashicorp/terraform/addrs"
|
||||
"github.com/hashicorp/terraform/internal/copydir"
|
||||
"github.com/hashicorp/terraform/internal/getproviders"
|
||||
)
|
||||
|
||||
// Installer is the main type in this package, representing a provider installer
|
||||
// with a particular configuration-specific cache directory and an optional
|
||||
// global cache directory.
|
||||
type Installer struct {
|
||||
// targetDir is the cache directory we're ultimately aiming to get the
|
||||
// requested providers installed into.
|
||||
targetDir *Dir
|
||||
|
||||
// source is the provider source that the installer will use to discover
|
||||
// what provider versions are available for installation and to
|
||||
// find the source locations for any versions that are not already
|
||||
// available via one of the cache directories.
|
||||
source getproviders.Source
|
||||
|
||||
// globalCacheDir is an optional additional directory that will, if
|
||||
// provided, be treated as a read-through cache when retrieving new
|
||||
// provider versions. That is, new packages are fetched into this
|
||||
// directory first and then linked into targetDir, which allows sharing
|
||||
// both the disk space and the download time for a particular provider
|
||||
// version between different configurations on the same system.
|
||||
globalCacheDir *Dir
|
||||
}
|
||||
|
||||
// NewInstaller constructs and returns a new installer with the given target
|
||||
// directory and provider source.
|
||||
//
|
||||
// A newly-created installer does not have a global cache directory configured,
|
||||
// but a caller can make a follow-up call to SetGlobalCacheDir to provide
|
||||
// one prior to taking any installation actions.
|
||||
//
|
||||
// The target directory MUST NOT also be an input consulted by the given source,
|
||||
// or the result is undefined.
|
||||
func NewInstaller(targetDir *Dir, source getproviders.Source) *Installer {
|
||||
return &Installer{
|
||||
targetDir: targetDir,
|
||||
source: source,
|
||||
}
|
||||
}
|
||||
|
||||
// SetGlobalCacheDir activates a second tier of caching for the receiving
|
||||
// installer, with the given directory used as a read-through cache for
|
||||
// installation operations that need to retrieve new packages.
|
||||
//
|
||||
// The global cache directory for an installer must never be the same as its
|
||||
// target directory, and must not be used as one of its provider sources.
|
||||
// If these overlap then undefined behavior will result.
|
||||
func (i *Installer) SetGlobalCacheDir(cacheDir *Dir) {
|
||||
// A little safety check to catch straightforward mistakes where the
|
||||
// directories overlap. Better to panic early than to do
|
||||
// possibly-distructive actions on the cache directory downstream.
|
||||
if same, err := copydir.SameFile(i.targetDir.baseDir, cacheDir.baseDir); err == nil && !same {
|
||||
panic(fmt.Sprintf("global cache directory %s must not match the installation target directory", i.targetDir.baseDir))
|
||||
}
|
||||
i.globalCacheDir = cacheDir
|
||||
}
|
||||
|
||||
// EnsureProviderVersions compares the given provider requirements with what
|
||||
// is already available in the installer's target directory and then takes
|
||||
// appropriate installation actions to ensure that suitable packages
|
||||
// are available in the target cache directory.
|
||||
//
|
||||
// The given mode modifies how the operation will treat providers that already
|
||||
// have acceptable versions available in the target cache directory. See the
|
||||
// documentation for InstallMode and the InstallMode values for more
|
||||
// information.
|
||||
//
|
||||
// The given context can be used to cancel the overall installation operation
|
||||
// (causing any operations in progress to fail with an error), and can also
|
||||
// include an InstallerEvents value for optional intermediate progress
|
||||
// notifications.
|
||||
//
|
||||
// If a given InstallerEvents subscribes to notifications about installation
|
||||
// failures then those notifications will be redundant with the ones included
|
||||
// in the final returned error value so callers should show either one or the
|
||||
// other, and not both.
|
||||
func (i *Installer) EnsureProviderVersions(ctx context.Context, reqs map[addrs.Provider]getproviders.VersionConstraints, mode InstallMode) (map[addrs.Provider]getproviders.Version, error) {
|
||||
// FIXME: Currently the context isn't actually propagated into the
|
||||
// other functions we call here, because they are not context-aware.
|
||||
// Right now the context is used only for the InstallerEvents object.
|
||||
// Before considering this "finished" we should update the functions
|
||||
// we're calling below that might perform external network requests
|
||||
// and make them also take a context and respect cancellation of it.
|
||||
|
||||
errs := map[addrs.Provider]error{}
|
||||
evts := installerEventsForContext(ctx)
|
||||
|
||||
if cb := evts.PendingProviders; cb != nil {
|
||||
cb(reqs)
|
||||
}
|
||||
|
||||
// Here we'll keep track of which exact version we've selected for each
|
||||
// provider in the requirements.
|
||||
selected := map[addrs.Provider]getproviders.Version{}
|
||||
|
||||
// Step 1: Which providers might we need to fetch a new version of?
|
||||
// This produces the subset of requirements we need to ask the provider
|
||||
// source about.
|
||||
have := i.targetDir.AllAvailablePackages()
|
||||
mightNeed := map[addrs.Provider]getproviders.VersionSet{}
|
||||
MightNeedProvider:
|
||||
for provider, versionConstraints := range reqs {
|
||||
acceptableVersions := versions.MeetingConstraints(versionConstraints)
|
||||
if mode.forceQueryAllProviders() {
|
||||
// If our mode calls for us to look for newer versions regardless
|
||||
// of whether an existing version is acceptable, we "might need"
|
||||
// _all_ of the requested providers.
|
||||
mightNeed[provider] = acceptableVersions
|
||||
continue
|
||||
}
|
||||
havePackages, ok := have[provider]
|
||||
if !ok { // If we don't have any versions at all then we'll definitely need it
|
||||
mightNeed[provider] = acceptableVersions
|
||||
continue
|
||||
}
|
||||
// If we already have some versions installed and our mode didn't
|
||||
// force us to check for new ones anyway then we'll check only if
|
||||
// there isn't already at least one version in our cache that is
|
||||
// in the set of acceptable versions.
|
||||
for _, pkg := range havePackages {
|
||||
if acceptableVersions.Has(pkg.Version) {
|
||||
// We will take no further actions for this provider, because
|
||||
// a version we have is already acceptable.
|
||||
selected[provider] = pkg.Version
|
||||
if cb := evts.ProviderAlreadyInstalled; cb != nil {
|
||||
cb(provider, pkg.Version)
|
||||
}
|
||||
continue MightNeedProvider
|
||||
}
|
||||
}
|
||||
// If we get here then we didn't find any cached version that is
|
||||
// in our set of acceptable versions.
|
||||
mightNeed[provider] = acceptableVersions
|
||||
}
|
||||
|
||||
// Step 2: Query the provider source for each of the providers we selected
|
||||
// in the first step and select the latest available version that is
|
||||
// in the set of acceptable versions.
|
||||
//
|
||||
// This produces a set of packages to install to our cache in the next step.
|
||||
need := map[addrs.Provider]getproviders.Version{}
|
||||
NeedProvider:
|
||||
for provider, acceptableVersions := range mightNeed {
|
||||
if cb := evts.QueryPackagesBegin; cb != nil {
|
||||
cb(provider, reqs[provider])
|
||||
}
|
||||
available, err := i.source.AvailableVersions(provider)
|
||||
if err != nil {
|
||||
// TODO: Consider retrying a few times for certain types of
|
||||
// source errors that seem likely to be transient.
|
||||
errs[provider] = err
|
||||
if cb := evts.QueryPackagesFailure; cb != nil {
|
||||
cb(provider, err)
|
||||
}
|
||||
// We will take no further actions for this provider.
|
||||
continue
|
||||
}
|
||||
available.Sort() // put the versions in increasing order of precedence
|
||||
for i := len(available) - 1; i >= 0; i-- { // walk backwards to consider newer versions first
|
||||
if acceptableVersions.Has(available[i]) {
|
||||
need[provider] = available[i]
|
||||
if cb := evts.QueryPackagesSuccess; cb != nil {
|
||||
cb(provider, available[i])
|
||||
}
|
||||
continue NeedProvider
|
||||
}
|
||||
}
|
||||
// If we get here then the source has no packages that meet the given
|
||||
// version constraint, which we model as a query error.
|
||||
err = fmt.Errorf("no available releases match the given constraints %s", getproviders.VersionConstraintsString(reqs[provider]))
|
||||
errs[provider] = err
|
||||
if cb := evts.QueryPackagesFailure; cb != nil {
|
||||
cb(provider, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: For each provider version we've decided we need to install,
|
||||
// install its package into our target cache (possibly via the global cache).
|
||||
targetPlatform := i.targetDir.targetPlatform // we inherit this to behave correctly in unit tests
|
||||
for provider, version := range need {
|
||||
if i.globalCacheDir != nil {
|
||||
// Step 3a: If our global cache already has this version available then
|
||||
// we'll just link it in.
|
||||
if cached := i.globalCacheDir.ProviderVersion(provider, version); cached != nil {
|
||||
if cb := evts.LinkFromCacheBegin; cb != nil {
|
||||
cb(provider, version, i.globalCacheDir.baseDir)
|
||||
}
|
||||
err := i.targetDir.LinkFromOtherCache(cached)
|
||||
if err != nil {
|
||||
errs[provider] = err
|
||||
if cb := evts.LinkFromCacheFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// We'll fetch what we just linked to make sure it actually
|
||||
// did show up there.
|
||||
new := i.targetDir.ProviderVersion(provider, version)
|
||||
if new == nil {
|
||||
err := fmt.Errorf("after linking %s from provider cache at %s it is still not detected in the target directory; this is a bug in Terraform", provider, i.globalCacheDir.baseDir)
|
||||
if cb := evts.LinkFromCacheFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
selected[provider] = version
|
||||
if cb := evts.LinkFromCacheSuccess; cb != nil {
|
||||
cb(provider, version, new.PackageDir)
|
||||
}
|
||||
continue // Don't need to do full install, then.
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3b: Get the package metadata for the selected version from our
|
||||
// provider source.
|
||||
//
|
||||
// This is the step where we might detect and report that the provider
|
||||
// isn't available for the current platform.
|
||||
if cb := evts.FetchPackageMeta; cb != nil {
|
||||
cb(provider, version)
|
||||
}
|
||||
meta, err := i.source.PackageMeta(provider, version, targetPlatform)
|
||||
if err != nil {
|
||||
errs[provider] = err
|
||||
if cb := evts.FetchPackageFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Step 3c: Retrieve the package indicated by the metadata we received,
|
||||
// either directly into our target directory or via the global cache
|
||||
// directory.
|
||||
if cb := evts.FetchPackageBegin; cb != nil {
|
||||
cb(provider, version, meta.Location)
|
||||
}
|
||||
var installTo, linkTo *Dir
|
||||
if i.globalCacheDir != nil {
|
||||
installTo = i.globalCacheDir
|
||||
linkTo = i.targetDir
|
||||
} else {
|
||||
installTo = i.targetDir
|
||||
linkTo = nil // no linking needed
|
||||
}
|
||||
err = installTo.InstallPackage(meta)
|
||||
if err != nil {
|
||||
// TODO: Consider retrying for certain kinds of error that seem
|
||||
// likely to be transient. For now, we just treat all errors equally.
|
||||
errs[provider] = err
|
||||
if cb := evts.FetchPackageFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
new := installTo.ProviderVersion(provider, version)
|
||||
if new == nil {
|
||||
err := fmt.Errorf("after installing %s it is still not detected in the target directory; this is a bug in Terraform", provider)
|
||||
if cb := evts.FetchPackageFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if linkTo != nil {
|
||||
// We skip emitting the "LinkFromCache..." events here because
|
||||
// it's simpler for the caller to treat them as mutually exclusive.
|
||||
// We can just subsume the linking step under the "FetchPackage..."
|
||||
// series here (and that's why we use FetchPackageFailure below).
|
||||
err := linkTo.LinkFromOtherCache(new)
|
||||
if err != nil {
|
||||
errs[provider] = err
|
||||
if cb := evts.FetchPackageFailure; cb != nil {
|
||||
cb(provider, version, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
selected[provider] = version
|
||||
if cb := evts.FetchPackageSuccess; cb != nil {
|
||||
cb(provider, version, new.PackageDir)
|
||||
}
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
return selected, InstallerError{
|
||||
ProviderErrors: errs,
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
// InstallMode customizes the details of how an install operation treats
|
||||
// providers that have versions already cached in the target directory.
|
||||
type InstallMode rune
|
||||
|
||||
const (
|
||||
// InstallNewProvidersOnly is an InstallMode that causes the installer
|
||||
// to accept any existing version of a requested provider that is already
|
||||
// cached as long as it's in the given version sets, without checking
|
||||
// whether new versions are available that are also in the given version
|
||||
// sets.
|
||||
InstallNewProvidersOnly InstallMode = 'N'
|
||||
|
||||
// InstallUpgrades is an InstallMode that causes the installer to check
|
||||
// all requested providers to see if new versions are available that
|
||||
// are also in the given version sets, even if a suitable version of
|
||||
// a given provider is already available.
|
||||
InstallUpgrades InstallMode = 'U'
|
||||
)
|
||||
|
||||
func (m InstallMode) forceQueryAllProviders() bool {
|
||||
return m == InstallUpgrades
|
||||
}
|
||||
|
||||
// InstallerError is an error type that may be returned (but is not guaranteed)
|
||||
// from Installer.EnsureProviderVersions to indicate potentially several
|
||||
// separate failed installation outcomes for different providers included in
|
||||
// the overall request.
|
||||
type InstallerError struct {
|
||||
ProviderErrors map[addrs.Provider]error
|
||||
}
|
||||
|
||||
func (err InstallerError) Error() string {
|
||||
addrs := make([]addrs.Provider, 0, len(err.ProviderErrors))
|
||||
for addr := range err.ProviderErrors {
|
||||
addrs = append(addrs, addr)
|
||||
}
|
||||
sort.Slice(addrs, func(i, j int) bool {
|
||||
return addrs[i].LessThan(addrs[j])
|
||||
})
|
||||
var b strings.Builder
|
||||
b.WriteString("some providers could not be installed:\n")
|
||||
for _, addr := range addrs {
|
||||
providerErr := err.ProviderErrors[addr]
|
||||
fmt.Fprintf(&b, "- %s: %s\n", addr, providerErr.Error())
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,11 +29,15 @@ type InstallerEvents struct {
|
|||
// A recipient driving a UI might, for example, use this to pre-allocate
|
||||
// UI space for status reports for all of the providers and then update
|
||||
// those positions in-place as other events arrive.
|
||||
PendingProviders func(provider addrs.Provider)
|
||||
PendingProviders func(reqs map[addrs.Provider]getproviders.VersionConstraints)
|
||||
|
||||
// ProviderAlreadyInstalled is called for any provider that was included
|
||||
// in PendingProviders but requires no further action because a suitable
|
||||
// version is already present in the local provider cache directory.
|
||||
//
|
||||
// This event can also appear after the QueryPackages... series if
|
||||
// querying determines that a version already available is the newest
|
||||
// available version.
|
||||
ProviderAlreadyInstalled func(provider addrs.Provider, selectedVersion getproviders.Version)
|
||||
|
||||
// The QueryPackages... family of events delimit the operation of querying
|
||||
|
|
@ -44,8 +48,13 @@ type InstallerEvents struct {
|
|||
// A particular install operation includes only one query per distinct
|
||||
// provider, so a caller can use the provider argument as a unique
|
||||
// identifier to correlate between successive events.
|
||||
QueryPackagesBegin func(provider addrs.Provider, versionSet getproviders.VersionSet)
|
||||
//
|
||||
// The Begin, Success, and Failure events will each occur only once per
|
||||
// distinct provider. The Retry event can occur zero or more times, and
|
||||
// signals a failure that the installer is considering transient.
|
||||
QueryPackagesBegin func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints)
|
||||
QueryPackagesSuccess func(provider addrs.Provider, selectedVersion getproviders.Version)
|
||||
QueryPackagesRetry func(provider addrs.Provider, err error)
|
||||
QueryPackagesFailure func(provider addrs.Provider, err error)
|
||||
|
||||
// The LinkFromCache... family of events delimit the operation of linking
|
||||
|
|
@ -75,8 +84,14 @@ type InstallerEvents struct {
|
|||
//
|
||||
// A particular provider will either notify the LinkFromCache... events
|
||||
// or the FetchPackage... events, never both in the same install operation.
|
||||
//
|
||||
// The Query, Begin, Success, and Failure events will each occur only once
|
||||
// per distinct provider. The Retry event can occur zero or more times, and
|
||||
// signals a failure that the installer is considering transient.
|
||||
FetchPackageMeta func(provider addrs.Provider, version getproviders.Version) // fetching metadata prior to real download
|
||||
FetchPackageBegin func(provider addrs.Provider, version getproviders.Version, location getproviders.PackageLocation)
|
||||
FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string)
|
||||
FetchPackageRetry func(provider addrs.Provider, version getproviders.Version, err error)
|
||||
FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue