From 2bd2a9a9238fc59eb60689ff06956590b68b5a63 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 25 Aug 2020 16:26:21 -0700 Subject: [PATCH] internal/getproviders: HTTPMirrorSource implementation We previously had this just stubbed out because it was a stretch goal for the v0.13.0 release and it ultimately didn't make it in. Here we fill out the existing stub -- with a minor change to its interface so it can access credentials -- with a client implementation that is compatible with the directory structure produced by the "terraform providers mirror" subcommand, were the result to be published on a static file server. --- internal/getproviders/errors.go | 28 ++ internal/getproviders/hash.go | 17 + internal/getproviders/http_mirror_source.go | 384 +++++++++++++++++- .../getproviders/http_mirror_source_test.go | 314 ++++++++++++++ internal/getproviders/registry_client_test.go | 6 +- provider_source.go | 2 +- .../docs/commands/cli-config.html.markdown | 15 + .../docs/commands/providers/mirror.html.md | 12 +- .../provider-network-mirror-protocol.html.md | 279 +++++++++++++ website/layouts/docs.erb | 4 + 10 files changed, 1048 insertions(+), 13 deletions(-) create mode 100644 internal/getproviders/http_mirror_source_test.go create mode 100644 website/docs/internals/provider-network-mirror-protocol.html.md diff --git a/internal/getproviders/errors.go b/internal/getproviders/errors.go index c5f5191e0..e93a76218 100644 --- a/internal/getproviders/errors.go +++ b/internal/getproviders/errors.go @@ -2,6 +2,7 @@ package getproviders import ( "fmt" + "net/url" svchost "github.com/hashicorp/terraform-svchost" "github.com/hashicorp/terraform/addrs" @@ -123,9 +124,23 @@ type ErrPlatformNotSupported struct { Provider addrs.Provider Version Version Platform Platform + + // MirrorURL, if non-nil, is the base URL of the mirror that serviced + // the request in place of the provider's origin registry. MirrorURL + // is nil for a direct query. + MirrorURL *url.URL } func (err ErrPlatformNotSupported) Error() string { + if err.MirrorURL != nil { + return fmt.Sprintf( + "provider mirror %s does not have a package of %s %s for %s", + err.MirrorURL.String(), + err.Provider, + err.Version, + err.Platform, + ) + } return fmt.Sprintf( "provider %s %s is not available for %s", err.Provider, @@ -166,9 +181,22 @@ func (err ErrProtocolNotSupported) Error() string { type ErrQueryFailed struct { Provider addrs.Provider Wrapped error + + // MirrorURL, if non-nil, is the base URL of the mirror that serviced + // the request in place of the provider's origin registry. MirrorURL + // is nil for a direct query. + MirrorURL *url.URL } func (err ErrQueryFailed) Error() string { + if err.MirrorURL != nil { + return fmt.Sprintf( + "failed to query provider mirror %s for %s: %s", + err.MirrorURL.String(), + err.Provider.String(), + err.Wrapped.Error(), + ) + } return fmt.Sprintf( "could not query provider registry for %s: %s", err.Provider.String(), diff --git a/internal/getproviders/hash.go b/internal/getproviders/hash.go index 504638471..f2d9b7b9d 100644 --- a/internal/getproviders/hash.go +++ b/internal/getproviders/hash.go @@ -51,6 +51,23 @@ func PackageMatchesHash(loc PackageLocation, want string) (bool, error) { } } +// PreferredHash examines all of the given hash strings and returns the one +// that the current version of Terraform considers to provide the strongest +// verification. +// +// Returns an empty string if none of the given hashes are of a supported +// format. If PreferredHash returns a non-empty string then it will be one +// of the hash strings in "given", and that hash is the one that must pass +// verification in order for a package to be considered valid. +func PreferredHash(given []string) string { + for _, s := range given { + if strings.HasPrefix(s, "h1:") { + return s + } + } + return "" +} + // PackageHashV1 computes a hash of the contents of the package at the given // location using hash algorithm 1. // diff --git a/internal/getproviders/http_mirror_source.go b/internal/getproviders/http_mirror_source.go index e56aab0eb..578c9ed7a 100644 --- a/internal/getproviders/http_mirror_source.go +++ b/internal/getproviders/http_mirror_source.go @@ -1,16 +1,33 @@ package getproviders import ( + "encoding/json" "fmt" + "io" + "log" + "mime" + "net/http" "net/url" + "path" + "strings" + + "github.com/hashicorp/go-retryablehttp" + svchost "github.com/hashicorp/terraform-svchost" + svcauth "github.com/hashicorp/terraform-svchost/auth" + "golang.org/x/net/idna" "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/helper/logging" + "github.com/hashicorp/terraform/httpclient" + "github.com/hashicorp/terraform/version" ) // HTTPMirrorSource is a source that reads provider metadata from a provider // mirror that is accessible over the HTTP provider mirror protocol. type HTTPMirrorSource struct { - baseURL *url.URL + baseURL *url.URL + creds svcauth.CredentialsSource + httpClient *retryablehttp.Client } var _ Source = (*HTTPMirrorSource)(nil) @@ -18,25 +35,380 @@ var _ Source = (*HTTPMirrorSource)(nil) // NewHTTPMirrorSource constructs and returns a new network mirror source with // the given base URL. The relative URL offsets defined by the HTTP mirror // protocol will be resolve relative to the given URL. -func NewHTTPMirrorSource(baseURL *url.URL) *HTTPMirrorSource { +// +// The given URL must use the "https" scheme, or this function will panic. +// (When the URL comes from user input, such as in the CLI config, it's the +// UI/config layer's responsibility to validate this and return a suitable +// error message for the end-user audience.) +func NewHTTPMirrorSource(baseURL *url.URL, creds svcauth.CredentialsSource) *HTTPMirrorSource { + httpClient := httpclient.New() + httpClient.Timeout = requestTimeout + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + // If we get redirected more than five times we'll assume we're + // in a redirect loop and bail out, rather than hanging forever. + if len(via) > 5 { + return fmt.Errorf("too many redirects") + } + return nil + } + return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient) +} + +func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource { + if baseURL.Scheme != "https" { + panic("non-https URL for HTTP mirror") + } + + // We borrow the retry settings and behaviors from the registry client, + // because our needs here are very similar to those of the registry client. + retryableClient := retryablehttp.NewClient() + retryableClient.HTTPClient = httpClient + retryableClient.RetryMax = discoveryRetry + retryableClient.RequestLogHook = requestLogHook + retryableClient.ErrorHandler = maxRetryErrorHandler + + logOutput, err := logging.LogOutput() + if err != nil { + log.Printf("[WARN] Failed to set up provider HTTP mirror logger, so continuing without client logging: %s", err) + } + retryableClient.Logger = log.New(logOutput, "", log.Flags()) + return &HTTPMirrorSource{ - baseURL: baseURL, + baseURL: baseURL, + creds: creds, + httpClient: retryableClient, } } // AvailableVersions retrieves the available versions for the given provider // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, Warnings, error) { - return nil, nil, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") + log.Printf("[DEBUG] Querying available versions of provider %s at network mirror %s", provider.String(), s.baseURL.String()) + + endpointPath := path.Join( + provider.Hostname.String(), + provider.Namespace, + provider.Type, + "index.json", + ) + + statusCode, body, finalURL, err := s.get(endpointPath) + defer func() { + if body != nil { + body.Close() + } + }() + if err != nil { + return nil, nil, s.errQueryFailed(provider, err) + } + + switch statusCode { + case http.StatusOK: + // Great! + case http.StatusNotFound: + return nil, nil, ErrProviderNotFound{ + Provider: provider, + } + case http.StatusUnauthorized, http.StatusForbidden: + return nil, nil, s.errUnauthorized(finalURL) + default: + return nil, nil, s.errQueryFailed(provider, fmt.Errorf("server returned unsuccessful status %d", statusCode)) + } + + // If we got here then the response had status OK and so our body + // will be non-nil and should contain some JSON for us to parse. + type ResponseBody struct { + Versions map[string]struct{} `json:"versions"` + } + var bodyContent ResponseBody + + dec := json.NewDecoder(body) + if err := dec.Decode(&bodyContent); err != nil { + return nil, nil, s.errQueryFailed(provider, fmt.Errorf("invalid response content from mirror server: %s", err)) + } + + if len(bodyContent.Versions) == 0 { + return nil, nil, nil + } + ret := make(VersionList, 0, len(bodyContent.Versions)) + for versionStr := range bodyContent.Versions { + version, err := ParseVersion(versionStr) + if err != nil { + log.Printf("[WARN] Ignoring invalid %s version string %q in provider mirror response", provider, versionStr) + continue + } + ret = append(ret, version) + } + + ret.Sort() + return ret, nil, nil } // PackageMeta retrieves metadata for the requested provider package // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { - return PackageMeta{}, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") + log.Printf("[DEBUG] Finding package URL for %s v%s on %s via network mirror %s", provider.String(), version.String(), target.String(), s.baseURL.String()) + + endpointPath := path.Join( + provider.Hostname.String(), + provider.Namespace, + provider.Type, + version.String()+".json", + ) + + statusCode, body, finalURL, err := s.get(endpointPath) + defer func() { + if body != nil { + body.Close() + } + }() + if err != nil { + return PackageMeta{}, s.errQueryFailed(provider, err) + } + + switch statusCode { + case http.StatusOK: + // Great! + case http.StatusNotFound: + // A 404 Not Found for a version we previously saw in index.json is + // a protocol error, so we'll report this as "query failed. + return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("provider mirror does not have archive index for previously-reported %s version %s", provider, version)) + case http.StatusUnauthorized, http.StatusForbidden: + return PackageMeta{}, s.errUnauthorized(finalURL) + default: + return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("server returned unsuccessful status %d", statusCode)) + } + + // If we got here then the response had status OK and so our body + // will be non-nil and should contain some JSON for us to parse. + type ResponseArchiveMeta struct { + RelativeURL string `json:"url"` + Hashes []string + } + type ResponseBody struct { + Archives map[string]*ResponseArchiveMeta `json:"archives"` + } + var bodyContent ResponseBody + + dec := json.NewDecoder(body) + if err := dec.Decode(&bodyContent); err != nil { + return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("invalid response content from mirror server: %s", err)) + } + + archiveMeta, ok := bodyContent.Archives[target.String()] + if !ok { + return PackageMeta{}, ErrPlatformNotSupported{ + Provider: provider, + Version: version, + Platform: target, + MirrorURL: s.baseURL, + } + } + + relURL, err := url.Parse(archiveMeta.RelativeURL) + if err != nil { + return PackageMeta{}, s.errQueryFailed( + provider, + fmt.Errorf("provider mirror returned invalid URL %q: %s", archiveMeta.RelativeURL, err), + ) + } + absURL := finalURL.ResolveReference(relURL) + + ret := PackageMeta{ + Provider: provider, + Version: version, + TargetPlatform: target, + + Location: PackageHTTPURL(absURL.String()), + Filename: path.Base(absURL.Path), + } + // A network mirror might not provide any hashes at all, in which case + // the package has no source-defined authentication whatsoever. + if len(archiveMeta.Hashes) > 0 { + ret.Authentication = NewPackageHashAuthentication(archiveMeta.Hashes) + } + + return ret, nil } // ForDisplay returns a string description of the source for user-facing output. func (s *HTTPMirrorSource) ForDisplay(provider addrs.Provider) string { - return "Network-based provider mirrors are not supported in this version of Terraform" + return "provider mirror at " + s.baseURL.String() +} + +// mirrorHost extracts the hostname portion of the configured base URL and +// returns it as a svchost.Hostname, normalized in the usual ways. +// +// If the returned error is non-nil then the given hostname doesn't comply +// with the IETF RFC 5891 section 5.3 and 5.4 validation rules, and thus cannot +// be interpreted as a valid Terraform service host. The IDNA validation errors +// are unfortunately usually not very user-friendly, but they are also +// relatively rare because the IDNA normalization rules are quite tolerant. +func (s *HTTPMirrorSource) mirrorHost() (svchost.Hostname, error) { + return svchostFromURL(s.baseURL) +} + +// mirrorHostCredentials returns the HostCredentials, if any, for the hostname +// included in the mirror base URL. +// +// It might return an error if the mirror base URL is invalid, or if the +// credentials lookup itself fails. +func (s *HTTPMirrorSource) mirrorHostCredentials() (svcauth.HostCredentials, error) { + hostname, err := s.mirrorHost() + if err != nil { + return nil, fmt.Errorf("invalid provider mirror base URL %s: %s", s.baseURL.String(), err) + } + + if s.creds == nil { + // No host-specific credentials, then. + return nil, nil + } + + return s.creds.ForHost(hostname) +} + +// get is the shared functionality for querying a JSON index from a mirror. +// +// It only handles the raw HTTP request. The "body" return value is the +// reader from the response if and only if the response status code is 200 OK +// and the Content-Type is application/json. In all other cases it's nil. +// If body is non-nil then the caller must close it after reading it. +// +// If the "finalURL" return value is not empty then it's the URL that actually +// produced the returned response, possibly after following some redirects. +func (s *HTTPMirrorSource) get(relativePath string) (statusCode int, body io.ReadCloser, finalURL *url.URL, error error) { + endpointPath, err := url.Parse(relativePath) + if err != nil { + // Should never happen because the caller should validate all of the + // components it's including in the path. + return 0, nil, nil, err + } + endpointURL := s.baseURL.ResolveReference(endpointPath) + + req, err := retryablehttp.NewRequest("GET", endpointURL.String(), nil) + if err != nil { + return 0, nil, endpointURL, err + } + req.Request.Header.Set(terraformVersionHeader, version.String()) + creds, err := s.mirrorHostCredentials() + if err != nil { + return 0, nil, endpointURL, fmt.Errorf("failed to determine request credentials: %s", err) + } + if creds != nil { + // Note that if the initial requests gets redirected elsewhere + // then the credentials will still be included in the new request, + // even if they are on a different hostname. This is intentional + // and consistent with how we handle credentials for other + // Terraform-native services, because the user model is to configure + // credentials for the "friendly hostname" they configured, not for + // whatever hostname ends up ultimately serving the request as an + // implementation detail. + creds.PrepareRequest(req.Request) + } + + resp, err := s.httpClient.Do(req) + if err != nil { + return 0, nil, endpointURL, err + } + defer func() { + // If we're not returning the body then we'll close it + // before we return. + if body == nil { + resp.Body.Close() + } + }() + // After this point, our final URL return value should always be the + // one from resp.Request, because that takes into account any redirects + // we followed along the way. + finalURL = resp.Request.URL + + if resp.StatusCode == http.StatusOK { + // If and only if we get an OK response, we'll check that the response + // type is JSON and return the body reader. + ct := resp.Header.Get("Content-Type") + mt, params, err := mime.ParseMediaType(ct) + if err != nil { + return 0, nil, finalURL, fmt.Errorf("response has invalid Content-Type: %s", err) + } + if mt != "application/json" { + return 0, nil, finalURL, fmt.Errorf("response has invalid Content-Type: must be application/json") + } + for name := range params { + // The application/json content-type has no defined parameters, + // but some servers are configured to include a redundant "charset" + // parameter anyway, presumably out of a sense of completeness. + // We'll ignore them but warn that we're ignoring them in case the + // subsequent parsing fails due to the server trying to use an + // unsupported character encoding. (RFC 7159 defines its own + // JSON-specific character encoding rules.) + log.Printf("[WARN] Network mirror returned %q as part of its JSON content type, which is not defined. Ignoring.", name) + } + body = resp.Body + } + + return resp.StatusCode, body, finalURL, nil +} + +func (s *HTTPMirrorSource) errQueryFailed(provider addrs.Provider, err error) error { + return ErrQueryFailed{ + Provider: provider, + Wrapped: err, + MirrorURL: s.baseURL, + } +} + +func (s *HTTPMirrorSource) errUnauthorized(finalURL *url.URL) error { + hostname, err := svchostFromURL(finalURL) + if err != nil { + // Again, weird but we'll tolerate it. + return fmt.Errorf("invalid credentials for %s", finalURL) + } + + return ErrUnauthorized{ + Hostname: hostname, + + // We can't easily tell from here whether we had credentials or + // not, so for now we'll just assume we did because "host rejected + // the given credentials" is, hopefully, still understandable in + // the event that there were none. (If this ends up being confusing + // in practice then we'll need to do some refactoring of how + // we handle credentials in this source.) + HaveCredentials: true, + } +} + +func svchostFromURL(u *url.URL) (svchost.Hostname, error) { + raw := u.Host + + // When "friendly hostnames" appear in Terraform-specific identifiers we + // typically constrain their syntax more strictly than the + // Internationalized Domain Name specifications call for, such as + // forbidding direct use of punycode, but in this case we're just + // working with a standard http: or https: URL and so we'll first use the + // IDNA "lookup" rules directly, with no additional notational constraints, + // to effectively normalize away the differences that would normally + // produce an error. + var portPortion string + if colonPos := strings.Index(raw, ":"); colonPos != -1 { + raw, portPortion = raw[:colonPos], raw[colonPos:] + } + // HTTPMirrorSource requires all URLs to be https URLs, because running + // a network mirror over HTTP would potentially transmit any configured + // credentials in cleartext. Therefore we don't need to do any special + // handling of default ports here, because svchost.Hostname already + // considers the absense of a port to represent the standard HTTPS port + // 443, and will normalize away an explicit specification of port 443 + // in svchost.ForComparison below. + + normalized, err := idna.Display.ToUnicode(raw) + if err != nil { + return svchost.Hostname(""), err + } + + // If ToUnicode succeeded above then "normalized" is now a hostname in the + // normalized IDNA form, with any direct punycode already interpreted and + // the case folding and other normalization rules applied. It should + // therefore now be accepted by svchost.ForComparison with no additional + // errors, but the port portion can still potentially be invalid. + return svchost.ForComparison(normalized + portPortion) } diff --git a/internal/getproviders/http_mirror_source_test.go b/internal/getproviders/http_mirror_source_test.go new file mode 100644 index 000000000..013672fbd --- /dev/null +++ b/internal/getproviders/http_mirror_source_test.go @@ -0,0 +1,314 @@ +package getproviders + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + svchost "github.com/hashicorp/terraform-svchost" + svcauth "github.com/hashicorp/terraform-svchost/auth" + + "github.com/hashicorp/terraform/addrs" +) + +func TestHTTPMirrorSource(t *testing.T) { + // For mirrors we require a HTTPS server, so we'll use httptest to create + // one. However, that means we need to instantiate the source in an unusual + // way to force it to use the test client that is configured to trust the + // test server. + httpServer := httptest.NewTLSServer(http.HandlerFunc(testHTTPMirrorSourceHandler)) + defer httpServer.Close() + httpClient := httpServer.Client() + baseURL, err := url.Parse(httpServer.URL) + if err != nil { + t.Fatalf("httptest.NewTLSServer returned a server with an invalid URL") + } + creds := svcauth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{ + svchost.Hostname(baseURL.Host): { + "token": "placeholder-token", + }, + }) + source := newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient) + + existingProvider := addrs.MustParseProviderSourceString("terraform.io/test/exists") + missingProvider := addrs.MustParseProviderSourceString("terraform.io/test/missing") + failingProvider := addrs.MustParseProviderSourceString("terraform.io/test/fails") + redirectingProvider := addrs.MustParseProviderSourceString("terraform.io/test/redirects") + redirectLoopProvider := addrs.MustParseProviderSourceString("terraform.io/test/redirect-loop") + tosPlatform := Platform{OS: "tos", Arch: "m68k"} + + t.Run("AvailableVersions for provider that exists", func(t *testing.T) { + got, _, err := source.AvailableVersions(existingProvider) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + want := VersionList{ + MustParseVersion("1.0.0"), + MustParseVersion("1.0.1"), + MustParseVersion("1.0.2-beta.1"), + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + t.Run("AvailableVersions for provider that doesn't exist", func(t *testing.T) { + _, _, err := source.AvailableVersions(missingProvider) + switch err := err.(type) { + case ErrProviderNotFound: + if got, want := err.Provider, missingProvider; got != want { + t.Errorf("wrong provider in error\ngot: %s\nwant: %s", got, want) + } + default: + t.Fatalf("wrong error type %T; want ErrProviderNotFound", err) + } + }) + t.Run("AvailableVersions without required credentials", func(t *testing.T) { + unauthSource := newHTTPMirrorSourceWithHTTPClient(baseURL, nil, httpClient) + _, _, err := unauthSource.AvailableVersions(existingProvider) + switch err := err.(type) { + case ErrUnauthorized: + if got, want := string(err.Hostname), baseURL.Host; got != want { + t.Errorf("wrong hostname in error\ngot: %s\nwant: %s", got, want) + } + default: + t.Fatalf("wrong error type %T; want ErrUnauthorized", err) + } + }) + t.Run("AvailableVersions when the response is a server error", func(t *testing.T) { + _, _, err := source.AvailableVersions(failingProvider) + switch err := err.(type) { + case ErrQueryFailed: + if got, want := err.Provider, failingProvider; got != want { + t.Errorf("wrong provider in error\ngot: %s\nwant: %s", got, want) + } + if err.MirrorURL != source.baseURL { + t.Errorf("error does not refer to the mirror URL") + } + default: + t.Fatalf("wrong error type %T; want ErrQueryFailed", err) + } + }) + t.Run("AvailableVersions for provider that redirects", func(t *testing.T) { + got, _, err := source.AvailableVersions(redirectingProvider) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + want := VersionList{ + MustParseVersion("1.0.0"), + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + t.Run("AvailableVersions for provider that redirects too much", func(t *testing.T) { + _, _, err := source.AvailableVersions(redirectLoopProvider) + if err == nil { + t.Fatalf("succeeded; expected error") + } + }) + t.Run("PackageMeta for a version that exists and has a hash", func(t *testing.T) { + version := MustParseVersion("1.0.0") + got, err := source.PackageMeta(existingProvider, version, tosPlatform) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + want := PackageMeta{ + Provider: existingProvider, + Version: version, + TargetPlatform: tosPlatform, + Filename: "terraform-provider-test_v1.0.0_tos_m68k.zip", + Location: PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.0_tos_m68k.zip"), + Authentication: packageHashAuthentication{ + RequiredHash: "h1:placeholder-hash", + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + t.Run("PackageMeta for a version that exists and has no hash", func(t *testing.T) { + version := MustParseVersion("1.0.1") + got, err := source.PackageMeta(existingProvider, version, tosPlatform) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + want := PackageMeta{ + Provider: existingProvider, + Version: version, + TargetPlatform: tosPlatform, + Filename: "terraform-provider-test_v1.0.1_tos_m68k.zip", + Location: PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.1_tos_m68k.zip"), + Authentication: nil, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + t.Run("PackageMeta for a version that exists but has no archives", func(t *testing.T) { + version := MustParseVersion("1.0.2-beta.1") + _, err := source.PackageMeta(existingProvider, version, tosPlatform) + switch err := err.(type) { + case ErrPlatformNotSupported: + if got, want := err.Provider, existingProvider; got != want { + t.Errorf("wrong provider in error\ngot: %s\nwant: %s", got, want) + } + if got, want := err.Platform, tosPlatform; got != want { + t.Errorf("wrong platform in error\ngot: %s\nwant: %s", got, want) + } + if err.MirrorURL != source.baseURL { + t.Errorf("error does not contain the mirror URL") + } + default: + t.Fatalf("wrong error type %T; want ErrPlatformNotSupported", err) + } + }) + t.Run("PackageMeta with redirect to a version that exists", func(t *testing.T) { + version := MustParseVersion("1.0.0") + got, err := source.PackageMeta(redirectingProvider, version, tosPlatform) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + want := PackageMeta{ + Provider: redirectingProvider, + Version: version, + TargetPlatform: tosPlatform, + Filename: "terraform-provider-test.zip", + + // NOTE: The final URL is interpreted relative to the redirect + // target, not relative to what we originally requested. + Location: PackageHTTPURL(httpServer.URL + "/redirect-target/terraform-provider-test.zip"), + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + t.Run("PackageMeta when the response is a server error", func(t *testing.T) { + version := MustParseVersion("1.0.0") + _, err := source.PackageMeta(failingProvider, version, tosPlatform) + switch err := err.(type) { + case ErrQueryFailed: + if got, want := err.Provider, failingProvider; got != want { + t.Errorf("wrong provider in error\ngot: %s\nwant: %s", got, want) + } + if err.MirrorURL != source.baseURL { + t.Errorf("error does not contain the mirror URL") + } + default: + t.Fatalf("wrong error type %T; want ErrQueryFailed", err) + } + }) +} + +func testHTTPMirrorSourceHandler(resp http.ResponseWriter, req *http.Request) { + if auth := req.Header.Get("authorization"); auth != "Bearer placeholder-token" { + resp.WriteHeader(401) + fmt.Fprintln(resp, "incorrect auth token") + } + + switch req.URL.Path { + case "/terraform.io/test/exists/index.json": + resp.Header().Add("Content-Type", "application/json; ignored=yes") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "versions": { + "1.0.0": {}, + "1.0.1": {}, + "1.0.2-beta.1": {} + } + } + `) + + case "/terraform.io/test/fails/index.json", "/terraform.io/test/fails/1.0.0.json": + resp.WriteHeader(500) + fmt.Fprint(resp, "server error") + + case "/terraform.io/test/exists/1.0.0.json": + resp.Header().Add("Content-Type", "application/json; ignored=yes") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "archives": { + "tos_m68k": { + "url": "terraform-provider-test_v1.0.0_tos_m68k.zip", + "hashes": [ + "h1:placeholder-hash" + ] + } + } + } + `) + + case "/terraform.io/test/exists/1.0.1.json": + resp.Header().Add("Content-Type", "application/json; ignored=yes") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "archives": { + "tos_m68k": { + "url": "terraform-provider-test_v1.0.1_tos_m68k.zip" + } + } + } + `) + + case "/terraform.io/test/exists/1.0.2-beta.1.json": + resp.Header().Add("Content-Type", "application/json; ignored=yes") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "archives": {} + } + `) + + case "/terraform.io/test/redirects/index.json": + resp.Header().Add("location", "/redirect-target/index.json") + resp.WriteHeader(301) + fmt.Fprint(resp, "redirect") + + case "/redirect-target/index.json": + resp.Header().Add("Content-Type", "application/json") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "versions": { + "1.0.0": {} + } + } + `) + + case "/terraform.io/test/redirects/1.0.0.json": + resp.Header().Add("location", "/redirect-target/1.0.0.json") + resp.WriteHeader(301) + fmt.Fprint(resp, "redirect") + + case "/redirect-target/1.0.0.json": + resp.Header().Add("Content-Type", "application/json") + resp.WriteHeader(200) + fmt.Fprint(resp, ` + { + "archives": { + "tos_m68k": { + "url": "terraform-provider-test.zip" + } + } + } + `) + + case "/terraform.io/test/redirect-loop/index.json": + // This is intentionally redirecting to itself, to create a loop. + resp.Header().Add("location", req.URL.Path) + resp.WriteHeader(301) + fmt.Fprint(resp, "redirect loop") + + default: + resp.WriteHeader(404) + fmt.Fprintln(resp, "not found") + } +} diff --git a/internal/getproviders/registry_client_test.go b/internal/getproviders/registry_client_test.go index f556583a3..aad11fd5d 100644 --- a/internal/getproviders/registry_client_test.go +++ b/internal/getproviders/registry_client_test.go @@ -89,7 +89,7 @@ func TestConfigureRegistryClientTimeout(t *testing.T) { }) } -// testServices starts up a local HTTP server running a fake provider registry +// testRegistryServices starts up a local HTTP server running a fake provider registry // service and returns a service discovery object pre-configured to consider // the host "example.com" to be served by the fake registry service. // @@ -103,7 +103,7 @@ func TestConfigureRegistryClientTimeout(t *testing.T) { // The second return value is a function to call at the end of a test function // to shut down the test server. After you call that function, the discovery // object becomes useless. -func testServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup func()) { +func testRegistryServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup func()) { server := httptest.NewServer(http.HandlerFunc(fakeRegistryHandler)) services = disco.New() @@ -141,7 +141,7 @@ func testServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup // As with testServices, the second return value is a function to call at the end // of your test in order to shut down the test server. func testRegistrySource(t *testing.T) (source *RegistrySource, baseURL string, cleanup func()) { - services, baseURL, close := testServices(t) + services, baseURL, close := testRegistryServices(t) source = NewRegistrySource(services) return source, baseURL, close } diff --git a/provider_source.go b/provider_source.go index bb06c0872..9f42a1d01 100644 --- a/provider_source.go +++ b/provider_source.go @@ -216,7 +216,7 @@ func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationLocati )) return nil, diags } - return getproviders.NewHTTPMirrorSource(url), nil + return getproviders.NewHTTPMirrorSource(url, services.CredentialsSource()), nil default: // We should not get here because the set of cases above should diff --git a/website/docs/commands/cli-config.html.markdown b/website/docs/commands/cli-config.html.markdown index 20f19a66b..7d0b96946 100644 --- a/website/docs/commands/cli-config.html.markdown +++ b/website/docs/commands/cli-config.html.markdown @@ -238,6 +238,21 @@ The following are the two supported installation method types: You can include multiple `filesystem_mirror` blocks in order to specify several different directories to search. +* `network_mirror`: consult a particular HTTPS server for copies of providers, + regardless of which registry host they belong to. This method requires the + additional argument `url` to indicate the mirror base URL, which should + use the `https:` scheme and end with a trailing slash. + + Terraform expects the given URL to be a base URL for an implementation of + [the provider network mirror protocol](/docs/internals/providor-network-mirror-protocol.html), + which is designed to be relatively easy to implement using typical static + website hosting mechanisms. + +~> **Warning:** Don't configure `network_mirror` URLs that you do not trust. +Provider mirror servers are subject to TLS certificate checks to verify +identity, but a network mirror with a TLS certificate can potentially serve +modified copies of upstream providers with malicious content. + Terraform will try all of the specified methods whose include and exclude patterns match a given provider, and select the newest version available across all of those methods that matches the version constraint given in each diff --git a/website/docs/commands/providers/mirror.html.md b/website/docs/commands/providers/mirror.html.md index 8d523f35f..4e418902d 100644 --- a/website/docs/commands/providers/mirror.html.md +++ b/website/docs/commands/providers/mirror.html.md @@ -36,9 +36,15 @@ Usage: `terraform providers mirror [options] ` A single target directory is required. Terraform will create under that directory the path structure that is expected for filesystem-based provider -plugin mirrors, populating it both with `.zip` files containing the plugins -themselves and `.json` index files that describe what is available in the -directory. +plugin mirrors, populating it with `.zip` files containing the plugins +themselves. + +Terraform will also generate various `.json` index files which contain suitable +responses to implement +[the network mirror protocol](/docs/internals/providor-network-mirror-protocol.html), +if you upload the resulting directory to a static website host. Terraform +ignores those index files when using the directory as a filesystem mirror, +because the directory entries themselves are authoritative in that case. This command supports the following additional option: diff --git a/website/docs/internals/provider-network-mirror-protocol.html.md b/website/docs/internals/provider-network-mirror-protocol.html.md new file mode 100644 index 000000000..3ad4895bb --- /dev/null +++ b/website/docs/internals/provider-network-mirror-protocol.html.md @@ -0,0 +1,279 @@ +--- +layout: "docs" +page_title: "Provider Network Mirror Protocol" +sidebar_current: "docs-internals-provider-network-mirror-protocol" +description: |- + The provider network mirror protocol is implemented by a server intending + to provide a mirror or read-through caching proxy for Terraform providers, + as an alternative distribution source from the provider's origin provider + registry. +--- + +# Provider Network Mirror Protocol + +-> Provider network mirrors are supported only in Terraform CLI v0.13.2 and later. Prior versions do not support this protocol. + +The provider network mirror protocol is an optional protocol which you can +implement to provide an alternative installation source for Terraform providers, +regardless of their origin registries. + +Terraform uses network mirrors only if you activate them explicitly in +[the CLI configuration's `provider_installation` block](/docs/commands/cli-config.html#provider-installation). +When enabled, a network mirror can serve providers belonging to any registry +hostname, which can allow an organization to serve all of the Terraform +providers they intend to use from an internal server, rather than from each +provider's origin registry. + +This is _not_ the protocol that should be implemented by a host intending to +serve as an origin registry for Terraform Providers. To provide an origin +registry (whose hostname would then be included in the source addresses of the +providers it hosts), implement +[the provider registry protocol](./provider-registry-protocol.html) +instead. + +## Provider Addresses + +Each Terraform provider has an associated address which uniquely identifies it +within Terraform. A provider address has the syntax `hostname/namespace/type`, +which is described in more detail in +[the Provider Requirements documentation](/docs/configuration/provider-requirements.html). + +By default, the `hostname` portion of a provider address serves both as part +of its unique identifier _and_ as the location of the registry to retrieve it +from. However, when you configure Terraform to install providers from a network +mirror, the `hostname` serves _only_ as an identifier and no longer as +an installation source. A provider mirror can therefore serve providers +belonging to a variety of different provider registry hostnames, including +providers from the public Terraform Registry at `registry.terraform.io`, from a +single server. + +In the relative URL patterns later in this document, the placeholder `:hostname` +refers to the hostname from the address of the provider being requested, not +the hostname where the provider network mirror is deployed. + +## Protocol Base URL + +Most Terraform-native services use +[the remote service discovery protocol](./remote-service-discovery.html) so +that the physical location of the endpoints can potentially be separated from +the hostname used in identifiers. The Provider Network Mirror protocol does +_not_ use the service discovery indirection, because a network mirror location +is only a physical location and is never used as part of the identifier of a +dependency in a Terraform configuration. + +Instead, the provider installation section of the CLI configuration accepts +a base URL directly. The given URL must use the scheme `https:`, and should +end with a trailing slash so that the relative URLs of the individual operation +endpoints will be resolved beneath it. + +```hcl +provider_installation { + network_mirror { + url = "https://terraform.example.com/providers/" + } +} +``` + +Terraform uses the base URL only as a stem to resolve the operation endpoint +URLs against, and so it will never access the base URL directly. You can +therefore, if desired, publish human-readable usage documentation for your +network mirror at that URL. + +The following sections describe the various operations that a provider +network mirror server must implement to be compatible with Terraform CLI's +provider installer. The indicated URLs are all relative to the given base URL, +as described above. + +The URLs are shown with the convention that a path portion with a colon `:` +prefix is a placeholder for a dynamically-selected value, while all other +path portions are literal. For example, in `:hostname/:namespace/:type/index.json`, +the first three path portions are placeholders while the third is literally +the string "index.json". + +The example requests in the following sections will assume the example mirror +base URL from the above CLI configuration example. + +### Authentication + +If the CLI configuration includes +[credentials](/docs/commands/cli-config.html#credentials) for the hostname +given in the network mirror base URL, Terraform will include those credentials +in its requests for operations described below. + +If the given URL uses a non-standard port number (other than 443) then the +credentials must be associated with a hostname that includes the port number, +such as `terraform.example.com:8443`. + +Terraform does _not_ send credentials when retrieving the archives whose +URLs are given in the "List Available Installation Packages" response below. +If a particular mirror considers the distribution packages themselves to be +sensitive then it must use cryptographically-secure, user-specific, and +time-limited URLs in the metadata response. Strategies for doing so are out +of scope of this protocol documentation. + +## List Available Versions + +This operation determines which versions are currently available for a +particular provider. + +| Method | Path | Produces | +|--------|-----------------------------------------|--------------------| +| `GET` | `:hostname/:namespace/:type/index.json` | `application/json` | + +### Parameters + +* `hostname` (required): the hostname portion of the address of the requested + provider. +* `namespace` (required): the namespace portion of the address of the requested + provider. +* `type` (required): the type portion of the address of the requested provider. + +### Sample Request + +``` +curl 'https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/index.json' +``` + +### Sample Response + +```json +{ + "versions": { + "2.0.0": {}, + "2.0.1": {} + } +} +``` + +### Response Properties + +A successful result is a JSON object containing a single property `versions`, +which must be a JSON object. + +Each of the property names of the `versions` object represents an available +version number. The property values must be objects, but no properties are +currently defined for those objects. Future versions of this protocol may +define optional per-version properties for Terraform to use as installation +hints, so implementations of the current version should leave those objects +empty. + +Return `404 Not Found` to signal that the mirror does not have a provider +with the given address. + +## List Available Installation Packages + +This operation returns download URLs and associated metadata for the +distribution packages for a particular version of a provider. + +Each distribution package is associated with a particular operating system +and architecture. A network mirror may host only a subset of the available +packages for a provider version, if the users of the mirror are known to all +use only a subset of the target platforms that Terraform supports. + +Terraform CLI uses this operation after it has selected the newest available +version matching the configured version constraints, in order to find a zip +archive containing the plugin itself. + +| Method | Path | Produces | +|--------|--------------------------------------------|--------------------| +| `GET` | `:hostname/:namespace/:type/:version.json` | `application/json` | + +### Parameters + +* `hostname` (required): the hostname portion of the address of the requested + provider. +* `namespace` (required): the namespace portion of the address of the requested + provider. +* `type` (required): the type portion of the address of the requested provider. +* `version` (required): the version selected to download. This will exactly + match one of the version strings returned from a previous call to + [List Available Versions](#list-available-versions). + +### Sample Request + +``` +curl 'https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/2.0.0.json' +``` + +### Sample Response + +```json +{ + "archives": { + "darwin_amd64": { + "url": "terraform-provider-random_2.0.0_darwin_amd64.zip", + "hashes": [ + "h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=" + ] + }, + "linux_amd64": { + "url": "terraform-provider-random_2.0.0_linux_amd64.zip", + "hashes": [ + "h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=" + ] + } + } +} +``` + +### Response Properties + +A successful result is a JSON object with a property called `archives`, which +must be a JSON object. + +Each of the property names of the `archives` object is a target platform +identifier, which consists of an operating system and architecture concatenated +with an underscore (`_`). + +Each property value in the `archives` object is itself a nested object with +the following properties: + +* `url` (required): a string specifying the URL from which Terraform should + download the `.zip` archive containing the requested provider plugin version. + + Terraform resolves the URL relative to the URL from which the current + JSON document was returned, so the examples above containing only a + filename would cause Terraform to construct a URL like: + + ``` + https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/terraform-provider-random_2.0.0_darwin_amd64.zip + ``` + +* `hashes` (optional): a JSON array of strings containing one or more hash + values for the indicated archive. These hashes use Terraform's provider + package hashing algorithm. At present, the easiest way to populate these + is to construct a mirror's JSON indices using the `terraform providers mirror` + command, as described in a later section, which will include the calculated + hashes of each provider. + + If the response includes at least one hash, Terraform will select the hash + whose algorithm it considers to be strongest and verify that the downloaded + package matches that hash. If the response does not include a `hashes` + property then Terraform will install the indicated archive with no + verification. + +Terraform CLI will only attempt to download versions that it has previously +seen in response to [List Available Versions](#list-available-versions). + +## Provider Mirror as a Static Website + +The provider mirror protocol is designed so that it can potentially implemented +by placing files on typical static website hosting services. When using this +strategy, implement the JSON index responses described above as `.json` files +in the appropriate nested subdirectories, and ensure that your system is +configured to serve `.json` files with the `application/json` media type. + +As a convenience, Terraform CLI includes +[the `terraform providers mirror` subcommand](https://www.terraform.io/docs/commands/providers/mirror.html), +which will analyze the current configuration for the providers it requires, +download the packages for those providers from their origin registries, and +place them into a local directory suitable for use as a mirror. + +The `terraform providers mirror` subcommand also generates `index.json` and +version-specific `.json` files that can, when placed in a static website hosting +system, produce responses compatible with the provider mirror protocol. + +If you wish to create a mirror with providers for a number of different +Terraform configurations, run `terraform providers mirror` in each configuration +in turn while providing the same output directory each time. Terraform will +then merge together all of the requirements into a single set of JSON indices. diff --git a/website/layouts/docs.erb b/website/layouts/docs.erb index 1d4e2ec8e..3f5d733bc 100644 --- a/website/layouts/docs.erb +++ b/website/layouts/docs.erb @@ -499,6 +499,10 @@ Module Registry Protocol + > + Provider Network Mirror Protocol + + > Provider Registry Protocol