internal/getproviders: HTTPMirrorSource implementation

We previously had this just stubbed out because it was a stretch goal for
the v0.13.0 release and it ultimately didn't make it in.

Here we fill out the existing stub -- with a minor change to its interface
so it can access credentials -- with a client implementation that is
compatible with the directory structure produced by the
"terraform providers mirror" subcommand, were the result to be published
on a static file server.

internal/getproviders/errors.go

@@ -2,6 +2,7 @@ package getproviders
 
 import (
 	"fmt"
+	"net/url"
 
 	svchost "github.com/hashicorp/terraform-svchost"
 	"github.com/hashicorp/terraform/addrs"
@@ -123,9 +124,23 @@ type ErrPlatformNotSupported struct {
 	Provider addrs.Provider
 	Version  Version
 	Platform Platform
+
+	// MirrorURL, if non-nil, is the base URL of the mirror that serviced
+	// the request in place of the provider's origin registry. MirrorURL
+	// is nil for a direct query.
+	MirrorURL *url.URL
 }
 
 func (err ErrPlatformNotSupported) Error() string {
+	if err.MirrorURL != nil {
+		return fmt.Sprintf(
+			"provider mirror %s does not have a package of %s %s for %s",
+			err.MirrorURL.String(),
+			err.Provider,
+			err.Version,
+			err.Platform,
+		)
+	}
 	return fmt.Sprintf(
 		"provider %s %s is not available for %s",
 		err.Provider,
@@ -166,9 +181,22 @@ func (err ErrProtocolNotSupported) Error() string {
 type ErrQueryFailed struct {
 	Provider addrs.Provider
 	Wrapped  error
+
+	// MirrorURL, if non-nil, is the base URL of the mirror that serviced
+	// the request in place of the provider's origin registry. MirrorURL
+	// is nil for a direct query.
+	MirrorURL *url.URL
 }
 
 func (err ErrQueryFailed) Error() string {
+	if err.MirrorURL != nil {
+		return fmt.Sprintf(
+			"failed to query provider mirror %s for %s: %s",
+			err.MirrorURL.String(),
+			err.Provider.String(),
+			err.Wrapped.Error(),
+		)
+	}
 	return fmt.Sprintf(
 		"could not query provider registry for %s: %s",
 		err.Provider.String(),

internal/getproviders/hash.go

@@ -51,6 +51,23 @@ func PackageMatchesHash(loc PackageLocation, want string) (bool, error) {
 	}
 }
 
+// PreferredHash examines all of the given hash strings and returns the one
+// that the current version of Terraform considers to provide the strongest
+// verification.
+//
+// Returns an empty string if none of the given hashes are of a supported
+// format. If PreferredHash returns a non-empty string then it will be one
+// of the hash strings in "given", and that hash is the one that must pass
+// verification in order for a package to be considered valid.
+func PreferredHash(given []string) string {
+	for _, s := range given {
+		if strings.HasPrefix(s, "h1:") {
+			return s
+		}
+	}
+	return ""
+}
+
 // PackageHashV1 computes a hash of the contents of the package at the given
 // location using hash algorithm 1.
 //

internal/getproviders/http_mirror_source.go

@@ -1,16 +1,33 @@
 package getproviders
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
+	"log"
+	"mime"
+	"net/http"
 	"net/url"
+	"path"
+	"strings"
+
+	"github.com/hashicorp/go-retryablehttp"
+	svchost "github.com/hashicorp/terraform-svchost"
+	svcauth "github.com/hashicorp/terraform-svchost/auth"
+	"golang.org/x/net/idna"
 
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/helper/logging"
+	"github.com/hashicorp/terraform/httpclient"
+	"github.com/hashicorp/terraform/version"
 )
 
 // HTTPMirrorSource is a source that reads provider metadata from a provider
 // mirror that is accessible over the HTTP provider mirror protocol.
 type HTTPMirrorSource struct {
-	baseURL *url.URL
+	baseURL    *url.URL
+	creds      svcauth.CredentialsSource
+	httpClient *retryablehttp.Client
 }
 
 var _ Source = (*HTTPMirrorSource)(nil)
@@ -18,25 +35,380 @@ var _ Source = (*HTTPMirrorSource)(nil)
 // NewHTTPMirrorSource constructs and returns a new network mirror source with
 // the given base URL. The relative URL offsets defined by the HTTP mirror
 // protocol will be resolve relative to the given URL.
-func NewHTTPMirrorSource(baseURL *url.URL) *HTTPMirrorSource {
+//
+// The given URL must use the "https" scheme, or this function will panic.
+// (When the URL comes from user input, such as in the CLI config, it's the
+// UI/config layer's responsibility to validate this and return a suitable
+// error message for the end-user audience.)
+func NewHTTPMirrorSource(baseURL *url.URL, creds svcauth.CredentialsSource) *HTTPMirrorSource {
+	httpClient := httpclient.New()
+	httpClient.Timeout = requestTimeout
+	httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		// If we get redirected more than five times we'll assume we're
+		// in a redirect loop and bail out, rather than hanging forever.
+		if len(via) > 5 {
+			return fmt.Errorf("too many redirects")
+		}
+		return nil
+	}
+	return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient)
+}
+
+func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource {
+	if baseURL.Scheme != "https" {
+		panic("non-https URL for HTTP mirror")
+	}
+
+	// We borrow the retry settings and behaviors from the registry client,
+	// because our needs here are very similar to those of the registry client.
+	retryableClient := retryablehttp.NewClient()
+	retryableClient.HTTPClient = httpClient
+	retryableClient.RetryMax = discoveryRetry
+	retryableClient.RequestLogHook = requestLogHook
+	retryableClient.ErrorHandler = maxRetryErrorHandler
+
+	logOutput, err := logging.LogOutput()
+	if err != nil {
+		log.Printf("[WARN] Failed to set up provider HTTP mirror logger, so continuing without client logging: %s", err)
+	}
+	retryableClient.Logger = log.New(logOutput, "", log.Flags())
+
 	return &HTTPMirrorSource{
-		baseURL: baseURL,
+		baseURL:    baseURL,
+		creds:      creds,
+		httpClient: retryableClient,
 	}
 }
 
 // AvailableVersions retrieves the available versions for the given provider
 // from the object's underlying HTTP mirror service.
 func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, Warnings, error) {
-	return nil, nil, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform")
+	log.Printf("[DEBUG] Querying available versions of provider %s at network mirror %s", provider.String(), s.baseURL.String())
+
+	endpointPath := path.Join(
+		provider.Hostname.String(),
+		provider.Namespace,
+		provider.Type,
+		"index.json",
+	)
+
+	statusCode, body, finalURL, err := s.get(endpointPath)
+	defer func() {
+		if body != nil {
+			body.Close()
+		}
+	}()
+	if err != nil {
+		return nil, nil, s.errQueryFailed(provider, err)
+	}
+
+	switch statusCode {
+	case http.StatusOK:
+		// Great!
+	case http.StatusNotFound:
+		return nil, nil, ErrProviderNotFound{
+			Provider: provider,
+		}
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return nil, nil, s.errUnauthorized(finalURL)
+	default:
+		return nil, nil, s.errQueryFailed(provider, fmt.Errorf("server returned unsuccessful status %d", statusCode))
+	}
+
+	// If we got here then the response had status OK and so our body
+	// will be non-nil and should contain some JSON for us to parse.
+	type ResponseBody struct {
+		Versions map[string]struct{} `json:"versions"`
+	}
+	var bodyContent ResponseBody
+
+	dec := json.NewDecoder(body)
+	if err := dec.Decode(&bodyContent); err != nil {
+		return nil, nil, s.errQueryFailed(provider, fmt.Errorf("invalid response content from mirror server: %s", err))
+	}
+
+	if len(bodyContent.Versions) == 0 {
+		return nil, nil, nil
+	}
+	ret := make(VersionList, 0, len(bodyContent.Versions))
+	for versionStr := range bodyContent.Versions {
+		version, err := ParseVersion(versionStr)
+		if err != nil {
+			log.Printf("[WARN] Ignoring invalid %s version string %q in provider mirror response", provider, versionStr)
+			continue
+		}
+		ret = append(ret, version)
+	}
+
+	ret.Sort()
+	return ret, nil, nil
 }
 
 // PackageMeta retrieves metadata for the requested provider package
 // from the object's underlying HTTP mirror service.
 func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
-	return PackageMeta{}, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform")
+	log.Printf("[DEBUG] Finding package URL for %s v%s on %s via network mirror %s", provider.String(), version.String(), target.String(), s.baseURL.String())
+
+	endpointPath := path.Join(
+		provider.Hostname.String(),
+		provider.Namespace,
+		provider.Type,
+		version.String()+".json",
+	)
+
+	statusCode, body, finalURL, err := s.get(endpointPath)
+	defer func() {
+		if body != nil {
+			body.Close()
+		}
+	}()
+	if err != nil {
+		return PackageMeta{}, s.errQueryFailed(provider, err)
+	}
+
+	switch statusCode {
+	case http.StatusOK:
+		// Great!
+	case http.StatusNotFound:
+		// A 404 Not Found for a version we previously saw in index.json is
+		// a protocol error, so we'll report this as "query failed.
+		return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("provider mirror does not have archive index for previously-reported %s version %s", provider, version))
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return PackageMeta{}, s.errUnauthorized(finalURL)
+	default:
+		return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("server returned unsuccessful status %d", statusCode))
+	}
+
+	// If we got here then the response had status OK and so our body
+	// will be non-nil and should contain some JSON for us to parse.
+	type ResponseArchiveMeta struct {
+		RelativeURL string `json:"url"`
+		Hashes      []string
+	}
+	type ResponseBody struct {
+		Archives map[string]*ResponseArchiveMeta `json:"archives"`
+	}
+	var bodyContent ResponseBody
+
+	dec := json.NewDecoder(body)
+	if err := dec.Decode(&bodyContent); err != nil {
+		return PackageMeta{}, s.errQueryFailed(provider, fmt.Errorf("invalid response content from mirror server: %s", err))
+	}
+
+	archiveMeta, ok := bodyContent.Archives[target.String()]
+	if !ok {
+		return PackageMeta{}, ErrPlatformNotSupported{
+			Provider:  provider,
+			Version:   version,
+			Platform:  target,
+			MirrorURL: s.baseURL,
+		}
+	}
+
+	relURL, err := url.Parse(archiveMeta.RelativeURL)
+	if err != nil {
+		return PackageMeta{}, s.errQueryFailed(
+			provider,
+			fmt.Errorf("provider mirror returned invalid URL %q: %s", archiveMeta.RelativeURL, err),
+		)
+	}
+	absURL := finalURL.ResolveReference(relURL)
+
+	ret := PackageMeta{
+		Provider:       provider,
+		Version:        version,
+		TargetPlatform: target,
+
+		Location: PackageHTTPURL(absURL.String()),
+		Filename: path.Base(absURL.Path),
+	}
+	// A network mirror might not provide any hashes at all, in which case
+	// the package has no source-defined authentication whatsoever.
+	if len(archiveMeta.Hashes) > 0 {
+		ret.Authentication = NewPackageHashAuthentication(archiveMeta.Hashes)
+	}
+
+	return ret, nil
 }
 
 // ForDisplay returns a string description of the source for user-facing output.
 func (s *HTTPMirrorSource) ForDisplay(provider addrs.Provider) string {
-	return "Network-based provider mirrors are not supported in this version of Terraform"
+	return "provider mirror at " + s.baseURL.String()
+}
+
+// mirrorHost extracts the hostname portion of the configured base URL and
+// returns it as a svchost.Hostname, normalized in the usual ways.
+//
+// If the returned error is non-nil then the given hostname doesn't comply
+// with the IETF RFC 5891 section 5.3 and 5.4 validation rules, and thus cannot
+// be interpreted as a valid Terraform service host. The IDNA validation errors
+// are unfortunately usually not very user-friendly, but they are also
+// relatively rare because the IDNA normalization rules are quite tolerant.
+func (s *HTTPMirrorSource) mirrorHost() (svchost.Hostname, error) {
+	return svchostFromURL(s.baseURL)
+}
+
+// mirrorHostCredentials returns the HostCredentials, if any, for the hostname
+// included in the mirror base URL.
+//
+// It might return an error if the mirror base URL is invalid, or if the
+// credentials lookup itself fails.
+func (s *HTTPMirrorSource) mirrorHostCredentials() (svcauth.HostCredentials, error) {
+	hostname, err := s.mirrorHost()
+	if err != nil {
+		return nil, fmt.Errorf("invalid provider mirror base URL %s: %s", s.baseURL.String(), err)
+	}
+
+	if s.creds == nil {
+		// No host-specific credentials, then.
+		return nil, nil
+	}
+
+	return s.creds.ForHost(hostname)
+}
+
+// get is the shared functionality for querying a JSON index from a mirror.
+//
+// It only handles the raw HTTP request. The "body" return value is the
+// reader from the response if and only if the response status code is 200 OK
+// and the Content-Type is application/json. In all other cases it's nil.
+// If body is non-nil then the caller must close it after reading it.
+//
+// If the "finalURL" return value is not empty then it's the URL that actually
+// produced the returned response, possibly after following some redirects.
+func (s *HTTPMirrorSource) get(relativePath string) (statusCode int, body io.ReadCloser, finalURL *url.URL, error error) {
+	endpointPath, err := url.Parse(relativePath)
+	if err != nil {
+		// Should never happen because the caller should validate all of the
+		// components it's including in the path.
+		return 0, nil, nil, err
+	}
+	endpointURL := s.baseURL.ResolveReference(endpointPath)
+
+	req, err := retryablehttp.NewRequest("GET", endpointURL.String(), nil)
+	if err != nil {
+		return 0, nil, endpointURL, err
+	}
+	req.Request.Header.Set(terraformVersionHeader, version.String())
+	creds, err := s.mirrorHostCredentials()
+	if err != nil {
+		return 0, nil, endpointURL, fmt.Errorf("failed to determine request credentials: %s", err)
+	}
+	if creds != nil {
+		// Note that if the initial requests gets redirected elsewhere
+		// then the credentials will still be included in the new request,
+		// even if they are on a different hostname. This is intentional
+		// and consistent with how we handle credentials for other
+		// Terraform-native services, because the user model is to configure
+		// credentials for the "friendly hostname" they configured, not for
+		// whatever hostname ends up ultimately serving the request as an
+		// implementation detail.
+		creds.PrepareRequest(req.Request)
+	}
+
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		return 0, nil, endpointURL, err
+	}
+	defer func() {
+		// If we're not returning the body then we'll close it
+		// before we return.
+		if body == nil {
+			resp.Body.Close()
+		}
+	}()
+	// After this point, our final URL return value should always be the
+	// one from resp.Request, because that takes into account any redirects
+	// we followed along the way.
+	finalURL = resp.Request.URL
+
+	if resp.StatusCode == http.StatusOK {
+		// If and only if we get an OK response, we'll check that the response
+		// type is JSON and return the body reader.
+		ct := resp.Header.Get("Content-Type")
+		mt, params, err := mime.ParseMediaType(ct)
+		if err != nil {
+			return 0, nil, finalURL, fmt.Errorf("response has invalid Content-Type: %s", err)
+		}
+		if mt != "application/json" {
+			return 0, nil, finalURL, fmt.Errorf("response has invalid Content-Type: must be application/json")
+		}
+		for name := range params {
+			// The application/json content-type has no defined parameters,
+			// but some servers are configured to include a redundant "charset"
+			// parameter anyway, presumably out of a sense of completeness.
+			// We'll ignore them but warn that we're ignoring them in case the
+			// subsequent parsing fails due to the server trying to use an
+			// unsupported character encoding. (RFC 7159 defines its own
+			// JSON-specific character encoding rules.)
+			log.Printf("[WARN] Network mirror returned %q as part of its JSON content type, which is not defined. Ignoring.", name)
+		}
+		body = resp.Body
+	}
+
+	return resp.StatusCode, body, finalURL, nil
+}
+
+func (s *HTTPMirrorSource) errQueryFailed(provider addrs.Provider, err error) error {
+	return ErrQueryFailed{
+		Provider:  provider,
+		Wrapped:   err,
+		MirrorURL: s.baseURL,
+	}
+}
+
+func (s *HTTPMirrorSource) errUnauthorized(finalURL *url.URL) error {
+	hostname, err := svchostFromURL(finalURL)
+	if err != nil {
+		// Again, weird but we'll tolerate it.
+		return fmt.Errorf("invalid credentials for %s", finalURL)
+	}
+
+	return ErrUnauthorized{
+		Hostname: hostname,
+
+		// We can't easily tell from here whether we had credentials or
+		// not, so for now we'll just assume we did because "host rejected
+		// the given credentials" is, hopefully, still understandable in
+		// the event that there were none. (If this ends up being confusing
+		// in practice then we'll need to do some refactoring of how
+		// we handle credentials in this source.)
+		HaveCredentials: true,
+	}
+}
+
+func svchostFromURL(u *url.URL) (svchost.Hostname, error) {
+	raw := u.Host
+
+	// When "friendly hostnames" appear in Terraform-specific identifiers we
+	// typically constrain their syntax more strictly than the
+	// Internationalized Domain Name specifications call for, such as
+	// forbidding direct use of punycode, but in this case we're just
+	// working with a standard http: or https: URL and so we'll first use the
+	// IDNA "lookup" rules directly, with no additional notational constraints,
+	// to effectively normalize away the differences that would normally
+	// produce an error.
+	var portPortion string
+	if colonPos := strings.Index(raw, ":"); colonPos != -1 {
+		raw, portPortion = raw[:colonPos], raw[colonPos:]
+	}
+	// HTTPMirrorSource requires all URLs to be https URLs, because running
+	// a network mirror over HTTP would potentially transmit any configured
+	// credentials in cleartext. Therefore we don't need to do any special
+	// handling of default ports here, because svchost.Hostname already
+	// considers the absense of a port to represent the standard HTTPS port
+	// 443, and will normalize away an explicit specification of port 443
+	// in svchost.ForComparison below.
+
+	normalized, err := idna.Display.ToUnicode(raw)
+	if err != nil {
+		return svchost.Hostname(""), err
+	}
+
+	// If ToUnicode succeeded above then "normalized" is now a hostname in the
+	// normalized IDNA form, with any direct punycode already interpreted and
+	// the case folding and other normalization rules applied. It should
+	// therefore now be accepted by svchost.ForComparison with no additional
+	// errors, but the port portion can still potentially be invalid.
+	return svchost.ForComparison(normalized + portPortion)
 }

internal/getproviders/http_mirror_source_test.go

@@ -0,0 +1,314 @@
+package getproviders
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	svchost "github.com/hashicorp/terraform-svchost"
+	svcauth "github.com/hashicorp/terraform-svchost/auth"
+
+	"github.com/hashicorp/terraform/addrs"
+)
+
+func TestHTTPMirrorSource(t *testing.T) {
+	// For mirrors we require a HTTPS server, so we'll use httptest to create
+	// one. However, that means we need to instantiate the source in an unusual
+	// way to force it to use the test client that is configured to trust the
+	// test server.
+	httpServer := httptest.NewTLSServer(http.HandlerFunc(testHTTPMirrorSourceHandler))
+	defer httpServer.Close()
+	httpClient := httpServer.Client()
+	baseURL, err := url.Parse(httpServer.URL)
+	if err != nil {
+		t.Fatalf("httptest.NewTLSServer returned a server with an invalid URL")
+	}
+	creds := svcauth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{
+		svchost.Hostname(baseURL.Host): {
+			"token": "placeholder-token",
+		},
+	})
+	source := newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient)
+
+	existingProvider := addrs.MustParseProviderSourceString("terraform.io/test/exists")
+	missingProvider := addrs.MustParseProviderSourceString("terraform.io/test/missing")
+	failingProvider := addrs.MustParseProviderSourceString("terraform.io/test/fails")
+	redirectingProvider := addrs.MustParseProviderSourceString("terraform.io/test/redirects")
+	redirectLoopProvider := addrs.MustParseProviderSourceString("terraform.io/test/redirect-loop")
+	tosPlatform := Platform{OS: "tos", Arch: "m68k"}
+
+	t.Run("AvailableVersions for provider that exists", func(t *testing.T) {
+		got, _, err := source.AvailableVersions(existingProvider)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		want := VersionList{
+			MustParseVersion("1.0.0"),
+			MustParseVersion("1.0.1"),
+			MustParseVersion("1.0.2-beta.1"),
+		}
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+	t.Run("AvailableVersions for provider that doesn't exist", func(t *testing.T) {
+		_, _, err := source.AvailableVersions(missingProvider)
+		switch err := err.(type) {
+		case ErrProviderNotFound:
+			if got, want := err.Provider, missingProvider; got != want {
+				t.Errorf("wrong provider in error\ngot:  %s\nwant: %s", got, want)
+			}
+		default:
+			t.Fatalf("wrong error type %T; want ErrProviderNotFound", err)
+		}
+	})
+	t.Run("AvailableVersions without required credentials", func(t *testing.T) {
+		unauthSource := newHTTPMirrorSourceWithHTTPClient(baseURL, nil, httpClient)
+		_, _, err := unauthSource.AvailableVersions(existingProvider)
+		switch err := err.(type) {
+		case ErrUnauthorized:
+			if got, want := string(err.Hostname), baseURL.Host; got != want {
+				t.Errorf("wrong hostname in error\ngot:  %s\nwant: %s", got, want)
+			}
+		default:
+			t.Fatalf("wrong error type %T; want ErrUnauthorized", err)
+		}
+	})
+	t.Run("AvailableVersions when the response is a server error", func(t *testing.T) {
+		_, _, err := source.AvailableVersions(failingProvider)
+		switch err := err.(type) {
+		case ErrQueryFailed:
+			if got, want := err.Provider, failingProvider; got != want {
+				t.Errorf("wrong provider in error\ngot:  %s\nwant: %s", got, want)
+			}
+			if err.MirrorURL != source.baseURL {
+				t.Errorf("error does not refer to the mirror URL")
+			}
+		default:
+			t.Fatalf("wrong error type %T; want ErrQueryFailed", err)
+		}
+	})
+	t.Run("AvailableVersions for provider that redirects", func(t *testing.T) {
+		got, _, err := source.AvailableVersions(redirectingProvider)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		want := VersionList{
+			MustParseVersion("1.0.0"),
+		}
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+	t.Run("AvailableVersions for provider that redirects too much", func(t *testing.T) {
+		_, _, err := source.AvailableVersions(redirectLoopProvider)
+		if err == nil {
+			t.Fatalf("succeeded; expected error")
+		}
+	})
+	t.Run("PackageMeta for a version that exists and has a hash", func(t *testing.T) {
+		version := MustParseVersion("1.0.0")
+		got, err := source.PackageMeta(existingProvider, version, tosPlatform)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		want := PackageMeta{
+			Provider:       existingProvider,
+			Version:        version,
+			TargetPlatform: tosPlatform,
+			Filename:       "terraform-provider-test_v1.0.0_tos_m68k.zip",
+			Location:       PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.0_tos_m68k.zip"),
+			Authentication: packageHashAuthentication{
+				RequiredHash: "h1:placeholder-hash",
+			},
+		}
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+	t.Run("PackageMeta for a version that exists and has no hash", func(t *testing.T) {
+		version := MustParseVersion("1.0.1")
+		got, err := source.PackageMeta(existingProvider, version, tosPlatform)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		want := PackageMeta{
+			Provider:       existingProvider,
+			Version:        version,
+			TargetPlatform: tosPlatform,
+			Filename:       "terraform-provider-test_v1.0.1_tos_m68k.zip",
+			Location:       PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.1_tos_m68k.zip"),
+			Authentication: nil,
+		}
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+	t.Run("PackageMeta for a version that exists but has no archives", func(t *testing.T) {
+		version := MustParseVersion("1.0.2-beta.1")
+		_, err := source.PackageMeta(existingProvider, version, tosPlatform)
+		switch err := err.(type) {
+		case ErrPlatformNotSupported:
+			if got, want := err.Provider, existingProvider; got != want {
+				t.Errorf("wrong provider in error\ngot:  %s\nwant: %s", got, want)
+			}
+			if got, want := err.Platform, tosPlatform; got != want {
+				t.Errorf("wrong platform in error\ngot:  %s\nwant: %s", got, want)
+			}
+			if err.MirrorURL != source.baseURL {
+				t.Errorf("error does not contain the mirror URL")
+			}
+		default:
+			t.Fatalf("wrong error type %T; want ErrPlatformNotSupported", err)
+		}
+	})
+	t.Run("PackageMeta with redirect to a version that exists", func(t *testing.T) {
+		version := MustParseVersion("1.0.0")
+		got, err := source.PackageMeta(redirectingProvider, version, tosPlatform)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		want := PackageMeta{
+			Provider:       redirectingProvider,
+			Version:        version,
+			TargetPlatform: tosPlatform,
+			Filename:       "terraform-provider-test.zip",
+
+			// NOTE: The final URL is interpreted relative to the redirect
+			// target, not relative to what we originally requested.
+			Location: PackageHTTPURL(httpServer.URL + "/redirect-target/terraform-provider-test.zip"),
+		}
+		if diff := cmp.Diff(want, got); diff != "" {
+			t.Errorf("wrong result\n%s", diff)
+		}
+	})
+	t.Run("PackageMeta when the response is a server error", func(t *testing.T) {
+		version := MustParseVersion("1.0.0")
+		_, err := source.PackageMeta(failingProvider, version, tosPlatform)
+		switch err := err.(type) {
+		case ErrQueryFailed:
+			if got, want := err.Provider, failingProvider; got != want {
+				t.Errorf("wrong provider in error\ngot:  %s\nwant: %s", got, want)
+			}
+			if err.MirrorURL != source.baseURL {
+				t.Errorf("error does not contain the mirror URL")
+			}
+		default:
+			t.Fatalf("wrong error type %T; want ErrQueryFailed", err)
+		}
+	})
+}
+
+func testHTTPMirrorSourceHandler(resp http.ResponseWriter, req *http.Request) {
+	if auth := req.Header.Get("authorization"); auth != "Bearer placeholder-token" {
+		resp.WriteHeader(401)
+		fmt.Fprintln(resp, "incorrect auth token")
+	}
+
+	switch req.URL.Path {
+	case "/terraform.io/test/exists/index.json":
+		resp.Header().Add("Content-Type", "application/json; ignored=yes")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"versions": {
+					"1.0.0": {},
+					"1.0.1": {},
+					"1.0.2-beta.1": {}
+				}
+			}
+		`)
+
+	case "/terraform.io/test/fails/index.json", "/terraform.io/test/fails/1.0.0.json":
+		resp.WriteHeader(500)
+		fmt.Fprint(resp, "server error")
+
+	case "/terraform.io/test/exists/1.0.0.json":
+		resp.Header().Add("Content-Type", "application/json; ignored=yes")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"archives": {
+					"tos_m68k": {
+						"url": "terraform-provider-test_v1.0.0_tos_m68k.zip",
+						"hashes": [
+							"h1:placeholder-hash"
+						]
+					}
+				}
+			}
+		`)
+
+	case "/terraform.io/test/exists/1.0.1.json":
+		resp.Header().Add("Content-Type", "application/json; ignored=yes")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"archives": {
+					"tos_m68k": {
+						"url": "terraform-provider-test_v1.0.1_tos_m68k.zip"
+					}
+				}
+			}
+		`)
+
+	case "/terraform.io/test/exists/1.0.2-beta.1.json":
+		resp.Header().Add("Content-Type", "application/json; ignored=yes")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"archives": {}
+			}
+		`)
+
+	case "/terraform.io/test/redirects/index.json":
+		resp.Header().Add("location", "/redirect-target/index.json")
+		resp.WriteHeader(301)
+		fmt.Fprint(resp, "redirect")
+
+	case "/redirect-target/index.json":
+		resp.Header().Add("Content-Type", "application/json")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"versions": {
+					"1.0.0": {}
+				}
+			}
+		`)
+
+	case "/terraform.io/test/redirects/1.0.0.json":
+		resp.Header().Add("location", "/redirect-target/1.0.0.json")
+		resp.WriteHeader(301)
+		fmt.Fprint(resp, "redirect")
+
+	case "/redirect-target/1.0.0.json":
+		resp.Header().Add("Content-Type", "application/json")
+		resp.WriteHeader(200)
+		fmt.Fprint(resp, `
+			{
+				"archives": {
+					"tos_m68k": {
+						"url": "terraform-provider-test.zip"
+					}
+				}
+			}
+		`)
+
+	case "/terraform.io/test/redirect-loop/index.json":
+		// This is intentionally redirecting to itself, to create a loop.
+		resp.Header().Add("location", req.URL.Path)
+		resp.WriteHeader(301)
+		fmt.Fprint(resp, "redirect loop")
+
+	default:
+		resp.WriteHeader(404)
+		fmt.Fprintln(resp, "not found")
+	}
+}

internal/getproviders/registry_client_test.go

@@ -89,7 +89,7 @@ func TestConfigureRegistryClientTimeout(t *testing.T) {
 	})
 }
 
-// testServices starts up a local HTTP server running a fake provider registry
+// testRegistryServices starts up a local HTTP server running a fake provider registry
 // service and returns a service discovery object pre-configured to consider
 // the host "example.com" to be served by the fake registry service.
 //
@@ -103,7 +103,7 @@ func TestConfigureRegistryClientTimeout(t *testing.T) {
 // The second return value is a function to call at the end of a test function
 // to shut down the test server. After you call that function, the discovery
 // object becomes useless.
-func testServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup func()) {
+func testRegistryServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup func()) {
 	server := httptest.NewServer(http.HandlerFunc(fakeRegistryHandler))
 
 	services = disco.New()
@@ -141,7 +141,7 @@ func testServices(t *testing.T) (services *disco.Disco, baseURL string, cleanup
 // As with testServices, the second return value is a function to call at the end
 // of your test in order to shut down the test server.
 func testRegistrySource(t *testing.T) (source *RegistrySource, baseURL string, cleanup func()) {
-	services, baseURL, close := testServices(t)
+	services, baseURL, close := testRegistryServices(t)
 	source = NewRegistrySource(services)
 	return source, baseURL, close
 }

provider_source.go

@@ -216,7 +216,7 @@ func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationLocati
 			))
 			return nil, diags
 		}
-		return getproviders.NewHTTPMirrorSource(url), nil
+		return getproviders.NewHTTPMirrorSource(url, services.CredentialsSource()), nil
 
 	default:
 		// We should not get here because the set of cases above should

website/docs/commands/cli-config.html.markdown

@@ -238,6 +238,21 @@ The following are the two supported installation method types:
     You can include multiple `filesystem_mirror` blocks in order to specify
     several different directories to search.
 
+* `network_mirror`: consult a particular HTTPS server for copies of providers,
+  regardless of which registry host they belong to. This method requires the
+  additional argument `url` to indicate the mirror base URL, which should
+  use the `https:` scheme and end with a trailing slash.
+
+    Terraform expects the given URL to be a base URL for an implementation of
+    [the provider network mirror protocol](/docs/internals/providor-network-mirror-protocol.html),
+    which is designed to be relatively easy to implement using typical static
+    website hosting mechanisms.
+
+~> **Warning:** Don't configure `network_mirror` URLs that you do not trust.
+Provider mirror servers are subject to TLS certificate checks to verify
+identity, but a network mirror with a TLS certificate can potentially serve
+modified copies of upstream providers with malicious content.
+
 Terraform will try all of the specified methods whose include and exclude
 patterns match a given provider, and select the newest version available across
 all of those methods that matches the version constraint given in each

website/docs/commands/providers/mirror.html.md

@@ -36,9 +36,15 @@ Usage: `terraform providers mirror [options] <target-dir>`
 
 A single target directory is required. Terraform will create under that
 directory the path structure that is expected for filesystem-based provider
-plugin mirrors, populating it both with `.zip` files containing the plugins
+plugin mirrors, populating it with `.zip` files containing the plugins
-themselves and `.json` index files that describe what is available in the
+themselves.
-directory.
+
+Terraform will also generate various `.json` index files which contain suitable
+responses to implement
+[the network mirror protocol](/docs/internals/providor-network-mirror-protocol.html),
+if you upload the resulting directory to a static website host. Terraform
+ignores those index files when using the directory as a filesystem mirror,
+because the directory entries themselves are authoritative in that case.
 
 This command supports the following additional option:
 

website/docs/internals/provider-network-mirror-protocol.html.md

@@ -0,0 +1,279 @@
+---
+layout: "docs"
+page_title: "Provider Network Mirror Protocol"
+sidebar_current: "docs-internals-provider-network-mirror-protocol"
+description: |-
+  The provider network mirror protocol is implemented by a server intending
+  to provide a mirror or read-through caching proxy for Terraform providers,
+  as an alternative distribution source from the provider's origin provider
+  registry.
+---
+
+# Provider Network Mirror Protocol
+
+-> Provider network mirrors are supported only in Terraform CLI v0.13.2 and later. Prior versions do not support this protocol.
+
+The provider network mirror protocol is an optional protocol which you can
+implement to provide an alternative installation source for Terraform providers,
+regardless of their origin registries.
+
+Terraform uses network mirrors only if you activate them explicitly in
+[the CLI configuration's `provider_installation` block](/docs/commands/cli-config.html#provider-installation).
+When enabled, a network mirror can serve providers belonging to any registry
+hostname, which can allow an organization to serve all of the Terraform
+providers they intend to use from an internal server, rather than from each
+provider's origin registry.
+
+This is _not_ the protocol that should be implemented by a host intending to
+serve as an origin registry for Terraform Providers. To provide an origin
+registry (whose hostname would then be included in the source addresses of the
+providers it hosts), implement
+[the provider registry protocol](./provider-registry-protocol.html)
+instead.
+
+## Provider Addresses
+
+Each Terraform provider has an associated address which uniquely identifies it
+within Terraform. A provider address has the syntax `hostname/namespace/type`,
+which is described in more detail in
+[the Provider Requirements documentation](/docs/configuration/provider-requirements.html).
+
+By default, the `hostname` portion of a provider address serves both as part
+of its unique identifier _and_ as the location of the registry to retrieve it
+from. However, when you configure Terraform to install providers from a network
+mirror, the `hostname` serves _only_ as an identifier and no longer as
+an installation source. A provider mirror can therefore serve providers
+belonging to a variety of different provider registry hostnames, including
+providers from the public Terraform Registry at `registry.terraform.io`, from a
+single server.
+
+In the relative URL patterns later in this document, the placeholder `:hostname`
+refers to the hostname from the address of the provider being requested, not
+the hostname where the provider network mirror is deployed.
+
+## Protocol Base URL
+
+Most Terraform-native services use
+[the remote service discovery protocol](./remote-service-discovery.html) so
+that the physical location of the endpoints can potentially be separated from
+the hostname used in identifiers. The Provider Network Mirror protocol does
+_not_ use the service discovery indirection, because a network mirror location
+is only a physical location and is never used as part of the identifier of a
+dependency in a Terraform configuration.
+
+Instead, the provider installation section of the CLI configuration accepts
+a base URL directly. The given URL must use the scheme `https:`, and should
+end with a trailing slash so that the relative URLs of the individual operation
+endpoints will be resolved beneath it.
+
+```hcl
+provider_installation {
+  network_mirror {
+    url = "https://terraform.example.com/providers/"
+  }
+}
+```
+
+Terraform uses the base URL only as a stem to resolve the operation endpoint
+URLs against, and so it will never access the base URL directly. You can
+therefore, if desired, publish human-readable usage documentation for your
+network mirror at that URL.
+
+The following sections describe the various operations that a provider
+network mirror server must implement to be compatible with Terraform CLI's
+provider installer. The indicated URLs are all relative to the given base URL,
+as described above.
+
+The URLs are shown with the convention that a path portion with a colon `:`
+prefix is a placeholder for a dynamically-selected value, while all other
+path portions are literal. For example, in `:hostname/:namespace/:type/index.json`,
+the first three path portions are placeholders while the third is literally
+the string "index.json".
+
+The example requests in the following sections will assume the example mirror
+base URL from the above CLI configuration example.
+
+### Authentication
+
+If the CLI configuration includes
+[credentials](/docs/commands/cli-config.html#credentials) for the hostname
+given in the network mirror base URL, Terraform will include those credentials
+in its requests for operations described below.
+
+If the given URL uses a non-standard port number (other than 443) then the
+credentials must be associated with a hostname that includes the port number,
+such as `terraform.example.com:8443`.
+
+Terraform does _not_ send credentials when retrieving the archives whose
+URLs are given in the "List Available Installation Packages" response below.
+If a particular mirror considers the distribution packages themselves to be
+sensitive then it must use cryptographically-secure, user-specific, and
+time-limited URLs in the metadata response. Strategies for doing so are out
+of scope of this protocol documentation.
+
+## List Available Versions
+
+This operation determines which versions are currently available for a
+particular provider.
+
+| Method | Path                                    | Produces           |
+|--------|-----------------------------------------|--------------------|
+| `GET`  | `:hostname/:namespace/:type/index.json` | `application/json` |
+
+### Parameters
+
+* `hostname` (required): the hostname portion of the address of the requested
+  provider.
+* `namespace` (required): the namespace portion of the address of the requested
+  provider.
+* `type` (required): the type portion of the address of the requested provider.
+
+### Sample Request
+
+```
+curl 'https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/index.json'
+```
+
+### Sample Response
+
+```json
+{
+  "versions": {
+    "2.0.0": {},
+    "2.0.1": {}
+  }
+}
+```
+
+### Response Properties
+
+A successful result is a JSON object containing a single property `versions`,
+which must be a JSON object.
+
+Each of the property names of the `versions` object represents an available
+version number. The property values must be objects, but no properties are
+currently defined for those objects. Future versions of this protocol may
+define optional per-version properties for Terraform to use as installation
+hints, so implementations of the current version should leave those objects
+empty.
+
+Return `404 Not Found` to signal that the mirror does not have a provider
+with the given address.
+
+## List Available Installation Packages
+
+This operation returns download URLs and associated metadata for the
+distribution packages for a particular version of a provider.
+
+Each distribution package is associated with a particular operating system
+and architecture. A network mirror may host only a subset of the available
+packages for a provider version, if the users of the mirror are known to all
+use only a subset of the target platforms that Terraform supports.
+
+Terraform CLI uses this operation after it has selected the newest available
+version matching the configured version constraints, in order to find a zip
+archive containing the plugin itself.
+
+| Method | Path                                       | Produces           |
+|--------|--------------------------------------------|--------------------|
+| `GET`  | `:hostname/:namespace/:type/:version.json` | `application/json` |
+
+### Parameters
+
+* `hostname` (required): the hostname portion of the address of the requested
+  provider.
+* `namespace` (required): the namespace portion of the address of the requested
+  provider.
+* `type` (required): the type portion of the address of the requested provider.
+* `version` (required): the version selected to download. This will exactly
+  match one of the version strings returned from a previous call to
+  [List Available Versions](#list-available-versions).
+
+### Sample Request
+
+```
+curl 'https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/2.0.0.json'
+```
+
+### Sample Response
+
+```json
+{
+  "archives": {
+    "darwin_amd64": {
+      "url": "terraform-provider-random_2.0.0_darwin_amd64.zip",
+      "hashes": [
+        "h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs="
+      ]
+    },
+    "linux_amd64": {
+      "url": "terraform-provider-random_2.0.0_linux_amd64.zip",
+      "hashes": [
+        "h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ="
+      ]
+    }
+  }
+}
+```
+
+### Response Properties
+
+A successful result is a JSON object with a property called `archives`, which
+must be a JSON object.
+
+Each of the property names of the `archives` object is a target platform
+identifier, which consists of an operating system and architecture concatenated
+with an underscore (`_`).
+
+Each property value in the `archives` object is itself a nested object with
+the following properties:
+
+* `url` (required): a string specifying the URL from which Terraform should
+  download the `.zip` archive containing the requested provider plugin version.
+
+    Terraform resolves the URL relative to the URL from which the current
+    JSON document was returned, so the examples above containing only a
+    filename would cause Terraform to construct a URL like:
+
+    ```
+    https://terraform.example.com/providers/registry.terraform.io/hashicorp/random/terraform-provider-random_2.0.0_darwin_amd64.zip
+    ```
+
+* `hashes` (optional): a JSON array of strings containing one or more hash
+  values for the indicated archive. These hashes use Terraform's provider
+  package hashing algorithm. At present, the easiest way to populate these
+  is to construct a mirror's JSON indices using the `terraform providers mirror`
+  command, as described in a later section, which will include the calculated
+  hashes of each provider.
+
+    If the response includes at least one hash, Terraform will select the hash
+    whose algorithm it considers to be strongest and verify that the downloaded
+    package matches that hash. If the response does not include a `hashes`
+    property then Terraform will install the indicated archive with no
+    verification.
+
+Terraform CLI will only attempt to download versions that it has previously
+seen in response to [List Available Versions](#list-available-versions).
+
+## Provider Mirror as a Static Website
+
+The provider mirror protocol is designed so that it can potentially implemented
+by placing files on typical static website hosting services. When using this
+strategy, implement the JSON index responses described above as `.json` files
+in the appropriate nested subdirectories, and ensure that your system is
+configured to serve `.json` files with the `application/json` media type.
+
+As a convenience, Terraform CLI includes
+[the `terraform providers mirror` subcommand](https://www.terraform.io/docs/commands/providers/mirror.html),
+which will analyze the current configuration for the providers it requires,
+download the packages for those providers from their origin registries, and
+place them into a local directory suitable for use as a mirror.
+
+The `terraform providers mirror` subcommand also generates `index.json` and
+version-specific `.json` files that can, when placed in a static website hosting
+system, produce responses compatible with the provider mirror protocol.
+
+If you wish to create a mirror with providers for a number of different
+Terraform configurations, run `terraform providers mirror` in each configuration
+in turn while providing the same output directory each time. Terraform will
+then merge together all of the requirements into a single set of JSON indices.

website/layouts/docs.erb

@@ -499,6 +499,10 @@
             <a href="/docs/internals/module-registry-protocol.html">Module Registry Protocol</a>
           </li>
 
+          <li<%= sidebar_current("docs-internals-provider-network-mirror-protocol") %>>
+            <a href="/docs/internals/provider-network-mirror-protocol.html">Provider Network Mirror Protocol</a>
+          </li>
+
           <li<%= sidebar_current("docs-internals-provider-registry-protocol") %>>
             <a href="/docs/internals/provider-registry-protocol.html">Provider Registry Protocol</a>
           </li>