internal/getproviders: Query a provider registry

Our existing provider installer was originally built to work with
releases.hashicorp.com and later retrofitted to talk to the official
Terraform Registry. It also assumes a flat namespace of providers.

We're starting a new one here, copying and adapting code from the old one
as necessary, so that we can build out this new API while retaining all
of the existing functionality and then cut over to this new implementation
in a later step.

Here we're creating a foundational component for the new installer, which
is a mechanism to query for the available versions and download locations
of a particular provider.

Subsequent commits in this package will introduce other Source
implementations for installing from network and filesystem mirrors.

internal/getproviders/doc.go

@@ -0,0 +1,11 @@
+// Package getproviders is the lowest-level provider automatic installation
+// functionality. It can answer questions about what providers and provider
+// versions are available in a registry, and it can retrieve the URL for
+// the distribution archive for a specific version of a specific provider
+// targeting a particular platform.
+//
+// This package is not responsible for choosing the best version to install
+// from a set of available versions, or for any signature verification of the
+// archives it fetches. Callers will use this package in conjunction with other
+// logic elsewhere in order to construct a full provider installer.
+package getproviders

internal/getproviders/errors.go

@@ -0,0 +1,147 @@
+package getproviders
+
+import (
+	"fmt"
+
+	svchost "github.com/hashicorp/terraform-svchost"
+	"github.com/hashicorp/terraform/addrs"
+)
+
+// ErrHostNoProviders is an error type used to indicate that a hostname given
+// in a provider address does not support the provider registry protocol.
+type ErrHostNoProviders struct {
+	Hostname svchost.Hostname
+
+	// HasOtherVersionis set to true if the discovery process detected
+	// declarations of services named "providers" whose version numbers did not
+	// match any version supported by the current version of Terraform.
+	//
+	// If this is set, it's helpful to hint to the user in an error message
+	// that the provider host may be expecting an older or a newer version
+	// of Terraform, rather than that it isn't a provider registry host at all.
+	HasOtherVersion bool
+}
+
+func (err ErrHostNoProviders) Error() string {
+	switch {
+	case err.HasOtherVersion:
+		return fmt.Sprintf("host %s does not support the provider registry protocol required by this Terraform version, but may be compatible with a different Terraform version", err.Hostname.ForDisplay())
+	default:
+		return fmt.Sprintf("host %s does not offer a Terraform provider registry", err.Hostname.ForDisplay())
+	}
+}
+
+// ErrHostUnreachable is an error type used to indicate that a hostname
+// given in a provider address did not resolve in DNS, did not respond to an
+// HTTPS request for service discovery, or otherwise failed to correctly speak
+// the service discovery protocol.
+type ErrHostUnreachable struct {
+	Hostname svchost.Hostname
+	Wrapped  error
+}
+
+func (err ErrHostUnreachable) Error() string {
+	return fmt.Sprintf("could not connect to %s: %s", err.Hostname.ForDisplay(), err.Wrapped.Error())
+}
+
+// Unwrap returns the underlying error that occurred when trying to reach the
+// indicated host.
+func (err ErrHostUnreachable) Unwrap() error {
+	return err.Wrapped
+}
+
+// ErrUnauthorized is an error type used to indicate that a hostname
+// given in a provider address returned a "401 Unauthorized" or "403 Forbidden"
+// error response when we tried to access it.
+type ErrUnauthorized struct {
+	Hostname svchost.Hostname
+
+	// HaveCredentials is true when the request that failed included some
+	// credentials, and thus it seems that those credentials were invalid.
+	// Conversely, HaveCredentials is false if the request did not include
+	// credentials at all, in which case it seems that credentials must be
+	// provided.
+	HaveCredentials bool
+}
+
+func (err ErrUnauthorized) Error() string {
+	switch {
+	case err.HaveCredentials:
+		return fmt.Sprintf("host %s rejected the given authentication credentials", err.Hostname)
+	default:
+		return fmt.Sprintf("host %s requires authentication credentials", err.Hostname)
+	}
+}
+
+// ErrProviderNotKnown is an error type used to indicate that the hostname
+// given in a provider address does appear to be a provider registry but that
+// registry does not know about the given provider namespace or type.
+//
+// A caller serving requests from an end-user should recognize this error type
+// and use it to produce user-friendly hints for common errors such as failing
+// to specify an explicit source for a provider not in the default namespace
+// (one not under registry.terraform.io/hashicorp/). The default error message
+// for this type is a direct description of the problem with no such hints,
+// because we expect that the caller will have better context to decide what
+// hints are appropriate, e.g. by looking at the configuration given by the
+// user.
+type ErrProviderNotKnown struct {
+	Provider addrs.Provider
+}
+
+func (err ErrProviderNotKnown) Error() string {
+	return fmt.Sprintf(
+		"provider registry %s does not have a provider named %s",
+		err.Provider.Hostname.ForDisplay(),
+		err.Provider,
+	)
+}
+
+// ErrPlatformNotSupported is an error type used to indicate that a particular
+// version of a provider isn't available for a particular target platform.
+//
+// This is returned when DownloadLocation encounters a 404 Not Found response
+// from the underlying registry, because it presumes that a caller will only
+// ask for the DownloadLocation for a version it already found the existence
+// of via AvailableVersions.
+type ErrPlatformNotSupported struct {
+	Provider addrs.Provider
+	Version  Version
+	Platform Platform
+}
+
+func (err ErrPlatformNotSupported) Error() string {
+	return fmt.Sprintf(
+		"provider %s %s is not available for %s",
+		err.Provider,
+		err.Version,
+		err.Platform,
+	)
+}
+
+// ErrQueryFailed is an error type used to indicate that the hostname given
+// in a provider address does appear to be a provider registry but that when
+// we queried it for metadata for the given provider the server returned an
+// unexpected error.
+//
+// This is used for any error responses other than "Not Found", which would
+// indicate the absense of a provider and is thus reported using
+// ErrProviderNotKnown instead.
+type ErrQueryFailed struct {
+	Provider addrs.Provider
+	Wrapped  error
+}
+
+func (err ErrQueryFailed) Error() string {
+	return fmt.Sprintf(
+		"could not query provider registry for %s: %s",
+		err.Provider.String(),
+		err.Wrapped.Error(),
+	)
+}
+
+// Unwrap returns the underlying error that occurred when trying to reach the
+// indicated host.
+func (err ErrQueryFailed) Unwrap() error {
+	return err.Wrapped
+}

internal/getproviders/registry_client.go

@@ -0,0 +1,251 @@
+package getproviders
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/apparentlymart/go-versions/versions"
+	svchost "github.com/hashicorp/terraform-svchost"
+	svcauth "github.com/hashicorp/terraform-svchost/auth"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/httpclient"
+	"github.com/hashicorp/terraform/version"
+)
+
+const terraformVersionHeader = "X-Terraform-Version"
+
+// registryClient is a client for the provider registry protocol that is
+// specialized only for the needs of this package. It's not intended as a
+// general registry API client.
+type registryClient struct {
+	baseURL *url.URL
+	creds   svcauth.HostCredentials
+
+	httpClient *http.Client
+}
+
+func newRegistryClient(baseURL *url.URL, creds svcauth.HostCredentials) *registryClient {
+	httpClient := httpclient.New()
+	httpClient.Timeout = 10 * time.Second
+
+	return &registryClient{
+		baseURL:    baseURL,
+		creds:      creds,
+		httpClient: httpClient,
+	}
+}
+
+// ProviderVersions returns the raw version strings produced by the registry
+// for the given provider.
+//
+// The returned error will be ErrProviderNotKnown if the registry responds
+// with 404 Not Found to indicate that the namespace or provider type are
+// not known, ErrUnauthorized if the registry responds with 401 or 403 status
+// codes, or ErrQueryFailed for any other protocol or operational problem.
+func (c *registryClient) ProviderVersions(addr addrs.Provider) ([]string, error) {
+	endpointPath, err := url.Parse(path.Join(addr.Namespace, addr.Type, "versions"))
+	if err != nil {
+		// Should never happen because we're constructing this from
+		// already-validated components.
+		return nil, err
+	}
+	endpointURL := c.baseURL.ResolveReference(endpointPath)
+
+	req, err := http.NewRequest("GET", endpointURL.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	c.addHeadersToRequest(req)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, c.errQueryFailed(addr, err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// Great!
+	case http.StatusNotFound:
+		return nil, ErrProviderNotKnown{
+			Provider: addr,
+		}
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return nil, c.errUnauthorized(addr.Hostname)
+	default:
+		return nil, c.errQueryFailed(addr, errors.New(resp.Status))
+	}
+
+	// We ignore everything except the version numbers here because our goal
+	// is to find out which versions are available _at all_. Which ones are
+	// compatible with the current Terraform becomes relevant only once we've
+	// selected one, at which point we'll return an error if the selected one
+	// is incompatible.
+	//
+	// We intentionally produce an error on incompatibility, rather than
+	// silently ignoring an incompatible version, in order to give the user
+	// explicit feedback about why their selection wasn't valid and allow them
+	// to decide whether to fix that by changing the selection or by some other
+	// action such as upgrading Terraform, using a different OS to run
+	// Terraform, etc. Changes that affect compatibility are considered
+	// breaking changes from a provider API standpoint, so provider teams
+	// should change compatibility only in new major versions.
+	type ResponseBody struct {
+		Versions []struct {
+			Version string `json:"version"`
+		} `json:"versions"`
+	}
+	var body ResponseBody
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&body); err != nil {
+		return nil, c.errQueryFailed(addr, err)
+	}
+
+	if len(body.Versions) == 0 {
+		return nil, nil
+	}
+
+	ret := make([]string, len(body.Versions))
+	for i, v := range body.Versions {
+		ret[i] = v.Version
+	}
+	return ret, nil
+}
+
+// PackageMeta returns metadata about a distribution package for a
+// provider.
+//
+// The returned error will be ErrPlatformNotSupported if the registry responds
+// with 404 Not Found, under the assumption that the caller previously checked
+// that the provider and version are valid. It will return ErrUnauthorized if
+// the registry responds with 401 or 403 status codes, or ErrQueryFailed for
+// any other protocol or operational problem.
+func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
+	endpointPath, err := url.Parse(path.Join(
+		provider.Namespace,
+		provider.Type,
+		version.String(),
+		"download",
+		target.OS,
+		target.Arch,
+	))
+	if err != nil {
+		// Should never happen because we're constructing this from
+		// already-validated components.
+		return PackageMeta{}, err
+	}
+	endpointURL := c.baseURL.ResolveReference(endpointPath)
+
+	req, err := http.NewRequest("GET", endpointURL.String(), nil)
+	if err != nil {
+		return PackageMeta{}, err
+	}
+	c.addHeadersToRequest(req)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return PackageMeta{}, c.errQueryFailed(provider, err)
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// Great!
+	case http.StatusNotFound:
+		return PackageMeta{}, ErrPlatformNotSupported{
+			Provider: provider,
+			Version:  version,
+			Platform: target,
+		}
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return PackageMeta{}, c.errUnauthorized(provider.Hostname)
+	default:
+		return PackageMeta{}, c.errQueryFailed(provider, errors.New(resp.Status))
+	}
+
+	type ResponseBody struct {
+		Protocols   []string `json:"protocols"`
+		OS          string   `json:"os"`
+		Arch        string   `json:"arch"`
+		Filename    string   `json:"filename"`
+		DownloadURL string   `json:"download_url"`
+		SHA256Sum   string   `json:"shasum"`
+
+		// TODO: Other metadata for signature checking
+	}
+	var body ResponseBody
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&body); err != nil {
+		return PackageMeta{}, c.errQueryFailed(provider, err)
+	}
+
+	var protoVersions VersionList
+	for _, versionStr := range body.Protocols {
+		v, err := versions.ParseVersion(versionStr)
+		if err != nil {
+			return PackageMeta{}, c.errQueryFailed(
+				provider,
+				fmt.Errorf("registry response includes invalid version string %q: %s", versionStr, err),
+			)
+		}
+		protoVersions = append(protoVersions, v)
+	}
+	protoVersions.Sort()
+
+	ret := PackageMeta{
+		ProtocolVersions: protoVersions,
+		TargetPlatform: Platform{
+			OS:   body.OS,
+			Arch: body.Arch,
+		},
+		Filename:    body.Filename,
+		DownloadURL: body.DownloadURL,
+		// SHA256Sum is populated below
+	}
+
+	if len(body.SHA256Sum) != len(ret.SHA256Sum)*2 {
+		return PackageMeta{}, c.errQueryFailed(
+			provider,
+			fmt.Errorf("registry response includes invalid SHA256 hash %q: %s", body.SHA256Sum, err),
+		)
+	}
+	_, err = hex.Decode(ret.SHA256Sum[:], []byte(body.SHA256Sum))
+	if err != nil {
+		return PackageMeta{}, c.errQueryFailed(
+			provider,
+			fmt.Errorf("registry response includes invalid SHA256 hash %q: %s", body.SHA256Sum, err),
+		)
+	}
+
+	return ret, nil
+}
+
+func (c *registryClient) addHeadersToRequest(req *http.Request) {
+	if c.creds != nil {
+		c.creds.PrepareRequest(req)
+	}
+	req.Header.Set(terraformVersionHeader, version.String())
+}
+
+func (c *registryClient) errQueryFailed(provider addrs.Provider, err error) error {
+	return ErrQueryFailed{
+		Provider: provider,
+		Wrapped:  err,
+	}
+}
+
+func (c *registryClient) errUnauthorized(hostname svchost.Hostname) error {
+	return ErrUnauthorized{
+		Hostname:        hostname,
+		HaveCredentials: c.creds != nil,
+	}
+}

internal/getproviders/registry_client_test.go

@@ -0,0 +1,148 @@
+package getproviders
+
+import (
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	svchost "github.com/hashicorp/terraform-svchost"
+	disco "github.com/hashicorp/terraform-svchost/disco"
+)
+
+// testServices starts up a local HTTP server running a fake provider registry
+// service and returns a service discovery object pre-configured to consider
+// the host "example.com" to be served by the fake registry service.
+//
+// The returned discovery object also knows the hostname "not.example.com"
+// which does not have a provider registry at all and "too-new.example.com"
+// which has a "providers.v99" service that is inoperable but could be useful
+// to test the error reporting for detecting an unsupported protocol version.
+// It also knows fails.example.com but it refers to an endpoint that doesn't
+// correctly speak HTTP, to simulate a protocol error.
+//
+// The second return value is a function to call at the end of a test function
+// to shut down the test server. After you call that function, the discovery
+// object becomes useless.
+func testServices(t *testing.T) (*disco.Disco, func()) {
+	server := httptest.NewServer(http.HandlerFunc(fakeRegistryHandler))
+
+	services := disco.New()
+	services.ForceHostServices(svchost.Hostname("example.com"), map[string]interface{}{
+		"providers.v1": server.URL + "/providers/v1/",
+	})
+	services.ForceHostServices(svchost.Hostname("not.example.com"), map[string]interface{}{})
+	services.ForceHostServices(svchost.Hostname("too-new.example.com"), map[string]interface{}{
+		// This service doesn't actually work; it's here only to be
+		// detected as "too new" by the discovery logic.
+		"providers.v99": server.URL + "/providers/v99/",
+	})
+	services.ForceHostServices(svchost.Hostname("fails.example.com"), map[string]interface{}{
+		"providers.v1": server.URL + "/fails-immediately/",
+	})
+
+	return services, func() {
+		server.Close()
+	}
+}
+
+// testRegistrySource is a wrapper around testServices that uses the created
+// discovery object to produce a Source instance that is ready to use with the
+// fake registry services.
+//
+// As with testServices, the second return value is a function to call at the end
+// of your test in order to shut down the test server.
+func testRegistrySource(t *testing.T) (*RegistrySource, func()) {
+	services, close := testServices(t)
+	source := NewRegistrySource(services)
+	return source, close
+}
+
+func fakeRegistryHandler(resp http.ResponseWriter, req *http.Request) {
+	path := req.URL.EscapedPath()
+	if strings.HasPrefix(path, "/fails-immediately/") {
+		// Here we take over the socket and just close it immediately, to
+		// simulate one possible way a server might not be an HTTP server.
+		hijacker, ok := resp.(http.Hijacker)
+		if !ok {
+			// Not hijackable, so we'll just fail normally.
+			// If this happens, tests relying on this will fail.
+			resp.WriteHeader(500)
+			resp.Write([]byte(`cannot hijack`))
+			return
+		}
+		conn, _, err := hijacker.Hijack()
+		if err != nil {
+			resp.WriteHeader(500)
+			resp.Write([]byte(`hijack failed`))
+			return
+		}
+		conn.Close()
+		return
+	}
+
+	if !strings.HasPrefix(path, "/providers/v1/") {
+		resp.WriteHeader(404)
+		resp.Write([]byte(`not a provider registry endpoint`))
+		return
+	}
+
+	pathParts := strings.Split(path, "/")[3:]
+	if len(pathParts) < 3 {
+		resp.WriteHeader(404)
+		resp.Write([]byte(`unexpected number of path parts`))
+		return
+	}
+	log.Printf("[TRACE] fake provider registry request for %#v", pathParts)
+
+	if pathParts[2] == "versions" {
+		if len(pathParts) != 3 {
+			resp.WriteHeader(404)
+			resp.Write([]byte(`extraneous path parts`))
+			return
+		}
+
+		switch pathParts[0] + "/" + pathParts[1] {
+		case "awesomesauce/happycloud":
+			resp.Header().Set("Content-Type", "application/json")
+			resp.WriteHeader(200)
+			// Note that these version numbers are intentionally misordered
+			// so we can test that the client-side code places them in the
+			// correct order (lowest precedence first).
+			resp.Write([]byte(`{"versions":[{"version":"1.2.0"}, {"version":"1.0.0"}]}`))
+		case "weaksauce/no-versions":
+			resp.Header().Set("Content-Type", "application/json")
+			resp.WriteHeader(200)
+			resp.Write([]byte(`{"versions":[]}`))
+		default:
+			resp.WriteHeader(404)
+			resp.Write([]byte(`unknown namespace or provider type`))
+		}
+		return
+	}
+
+	if len(pathParts) == 6 && pathParts[3] == "download" {
+		switch pathParts[0] + "/" + pathParts[1] {
+		case "awesomesauce/happycloud":
+			if pathParts[4] == "nonexist" {
+				resp.WriteHeader(404)
+				resp.Write([]byte(`unsupported OS`))
+				return
+			}
+			resp.Header().Set("Content-Type", "application/json")
+			resp.WriteHeader(200)
+			// Note that these version numbers are intentionally misordered
+			// so we can test that the client-side code places them in the
+			// correct order (lowest precedence first).
+			resp.Write([]byte(`{"protocols":["5.0"],"os":"` + pathParts[4] + `","arch":"` + pathParts[5] + `","filename":"happycloud_` + pathParts[2] + `.zip","download_url":"/pkg/happycloud_` + pathParts[2] + `.zip","shasum":"000000000000000000000000000000000000000000000000000000000000f00d"}`))
+		default:
+			resp.WriteHeader(404)
+			resp.Write([]byte(`unknown namespace/provider/version/architecture`))
+		}
+		return
+	}
+
+	resp.WriteHeader(404)
+	resp.Write([]byte(`unrecognized path scheme`))
+}

internal/getproviders/registry_source.go

@@ -0,0 +1,130 @@
+package getproviders
+
+import (
+	"fmt"
+
+	svchost "github.com/hashicorp/terraform-svchost"
+	disco "github.com/hashicorp/terraform-svchost/disco"
+
+	"github.com/hashicorp/terraform/addrs"
+)
+
+// RegistrySource is a Source that knows how to find and install providers from
+// their originating provider registries.
+type RegistrySource struct {
+	services *disco.Disco
+}
+
+// NewRegistrySource creates and returns a new source that will install
+// providers from their originating provider registries.
+func NewRegistrySource(services *disco.Disco) *RegistrySource {
+	return &RegistrySource{
+		services: services,
+	}
+}
+
+// AvailableVersions returns all of the versions available for the provider
+// with the given address, or an error if that result cannot be determined.
+//
+// If the request fails, the returned error might be an value of
+// ErrHostNoProviders, ErrHostUnreachable, ErrUnauthenticated,
+// ErrProviderNotKnown, or ErrQueryFailed. Callers must be defensive and
+// expect errors of other types too, to allow for future expansion.
+func (s *RegistrySource) AvailableVersions(provider addrs.Provider) (VersionList, error) {
+	client, err := s.registryClient(provider.Hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	versionStrs, err := client.ProviderVersions(provider)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(versionStrs) == 0 {
+		return nil, nil
+	}
+
+	ret := make(VersionList, len(versionStrs))
+	for i, str := range versionStrs {
+		v, err := ParseVersion(str)
+		if err != nil {
+			return nil, ErrQueryFailed{
+				Provider: provider,
+				Wrapped:  fmt.Errorf("registry response includes invalid version string %q: %s", str, err),
+			}
+		}
+		ret[i] = v
+	}
+	ret.Sort() // lowest precedence first, preserving order when equal precedence
+	return ret, nil
+}
+
+// DownloadLocation returns metadata about the location and capabilities of
+// a distribution package for a particular provider at a particular version
+// targeting a particular platform.
+//
+// Callers of DownloadLocation should first call AvailableVersions and pass
+// one of the resulting versions to this function. This function cannot
+// distinguish between a version that is not available and an unsupported
+// target platform, so if it encounters either case it will return an error
+// suggesting that the target platform isn't supported under the assumption
+// that the caller already checked that the version is available at all.
+//
+// To find a package suitable for the platform where the provider installation
+// process is running, set the "target" argument to
+// findproviders.CurrentPlatform.
+//
+// If the request fails, the returned error might be an value of
+// ErrHostNoProviders, ErrHostUnreachable, ErrUnauthenticated,
+// ErrPlatformNotSupported, or ErrQueryFailed. Callers must be defensive and
+// expect errors of other types too, to allow for future expansion.
+func (s *RegistrySource) DownloadLocation(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
+	client, err := s.registryClient(provider.Hostname)
+	if err != nil {
+		return PackageMeta{}, err
+	}
+
+	return client.PackageMeta(provider, version, target)
+}
+
+func (s *RegistrySource) registryClient(hostname svchost.Hostname) (*registryClient, error) {
+	host, err := s.services.Discover(hostname)
+	if err != nil {
+		return nil, ErrHostUnreachable{
+			Hostname: hostname,
+			Wrapped:  err,
+		}
+	}
+
+	url, err := host.ServiceURL("providers.v1")
+	switch err := err.(type) {
+	case nil:
+		// okay! We'll fall through and return below.
+	case *disco.ErrServiceNotProvided:
+		return nil, ErrHostNoProviders{
+			Hostname: hostname,
+		}
+	case *disco.ErrVersionNotSupported:
+		return nil, ErrHostNoProviders{
+			Hostname:        hostname,
+			HasOtherVersion: true,
+		}
+	default:
+		return nil, ErrHostUnreachable{
+			Hostname: hostname,
+			Wrapped:  err,
+		}
+	}
+
+	// Check if we have credentials configured for this hostname.
+	creds, err := s.services.CredentialsForHost(hostname)
+	if err != nil {
+		// This indicates that a credentials helper failed, which means we
+		// can't do anything better than just pass through the helper's
+		// own error message.
+		return nil, fmt.Errorf("failed to retrieve credentials for %s: %s", hostname, err)
+	}
+
+	return newRegistryClient(url, creds), nil
+}

internal/getproviders/registry_source_test.go

@@ -0,0 +1,207 @@
+package getproviders
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/apparentlymart/go-versions/versions"
+	"github.com/google/go-cmp/cmp"
+	svchost "github.com/hashicorp/terraform-svchost"
+
+	"github.com/hashicorp/terraform/addrs"
+)
+
+func TestSourceAvailableVersions(t *testing.T) {
+	source, close := testRegistrySource(t)
+	defer close()
+
+	tests := []struct {
+		provider     string
+		wantVersions []string
+		wantErr      string
+	}{
+		// These test cases are relying on behaviors of the fake provider
+		// registry server implemented in client_test.go.
+		{
+			"example.com/awesomesauce/happycloud",
+			[]string{"1.0.0", "1.2.0"},
+			``,
+		},
+		{
+			"example.com/weaksauce/no-versions",
+			nil,
+			``, // having no versions is not an error, it's just odd
+		},
+		{
+			"example.com/nonexist/nonexist",
+			nil,
+			`provider registry example.com does not have a provider named example.com/nonexist/nonexist`,
+		},
+		{
+			"not.example.com/foo/bar",
+			nil,
+			`host not.example.com does not offer a Terraform provider registry`,
+		},
+		{
+			"too-new.example.com/foo/bar",
+			nil,
+			`host too-new.example.com does not support the provider registry protocol required by this Terraform version, but may be compatible with a different Terraform version`,
+		},
+		{
+			"fails.example.com/foo/bar",
+			nil,
+			`could not query provider registry for fails.example.com/foo/bar: Get http://placeholder-origin/fails-immediately/foo/bar/versions: EOF`,
+		},
+	}
+
+	// Sometimes error messages contain specific HTTP endpoint URLs, but
+	// since our test server is on a random port we'd not be able to
+	// consistently match those. Instead, we'll normalize the URLs.
+	urlPattern := regexp.MustCompile(`http://[^/]+/`)
+
+	for _, test := range tests {
+		t.Run(test.provider, func(t *testing.T) {
+			// TEMP: We don't yet have a function for parsing provider
+			// source addresses so we'll just fake it in here for now.
+			parts := strings.Split(test.provider, "/")
+			providerAddr := addrs.Provider{
+				Hostname:  svchost.Hostname(parts[0]),
+				Namespace: parts[1],
+				Type:      parts[2],
+			}
+
+			gotVersions, err := source.AvailableVersions(providerAddr)
+
+			if err != nil {
+				if test.wantErr == "" {
+					t.Fatalf("wrong error\ngot:  %s\nwant: <nil>", err.Error())
+				}
+				gotErr := urlPattern.ReplaceAllLiteralString(err.Error(), "http://placeholder-origin/")
+				if got, want := gotErr, test.wantErr; got != want {
+					t.Fatalf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+				return
+			}
+
+			if test.wantErr != "" {
+				t.Fatalf("wrong error\ngot:  <nil>\nwant: %s", test.wantErr)
+			}
+
+			var gotVersionsStr []string
+			if gotVersions != nil {
+				gotVersionsStr = make([]string, len(gotVersions))
+				for i, v := range gotVersions {
+					gotVersionsStr[i] = v.String()
+				}
+			}
+
+			if diff := cmp.Diff(test.wantVersions, gotVersionsStr); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+	}
+
+}
+
+func TestSourceDownloadLocation(t *testing.T) {
+	source, close := testRegistrySource(t)
+	defer close()
+
+	tests := []struct {
+		provider string
+		version  string
+		os, arch string
+		want     PackageMeta
+		wantErr  string
+	}{
+		// These test cases are relying on behaviors of the fake provider
+		// registry server implemented in client_test.go.
+		{
+			"example.com/awesomesauce/happycloud",
+			"1.2.0",
+			"linux", "amd64",
+			PackageMeta{
+				ProtocolVersions: VersionList{versions.MustParseVersion("5.0.0")},
+				TargetPlatform:   Platform{"linux", "amd64"},
+				Filename:         "happycloud_1.2.0.zip",
+				DownloadURL:      "/pkg/happycloud_1.2.0.zip",
+				SHA256Sum:        [32]uint8{30: 0xf0, 31: 0x0d}, // fake registry uses a memorable sum
+			},
+			``,
+		},
+		{
+			"example.com/awesomesauce/happycloud",
+			"1.2.0",
+			"nonexist", "amd64",
+			PackageMeta{},
+			`provider example.com/awesomesauce/happycloud 1.2.0 is not available for nonexist_amd64`,
+		},
+		{
+			"not.example.com/awesomesauce/happycloud",
+			"1.2.0",
+			"linux", "amd64",
+			PackageMeta{},
+			`host not.example.com does not offer a Terraform provider registry`,
+		},
+		{
+			"too-new.example.com/awesomesauce/happycloud",
+			"1.2.0",
+			"linux", "amd64",
+			PackageMeta{},
+			`host too-new.example.com does not support the provider registry protocol required by this Terraform version, but may be compatible with a different Terraform version`,
+		},
+		{
+			"fails.example.com/awesomesauce/happycloud",
+			"1.2.0",
+			"linux", "amd64",
+			PackageMeta{},
+			`could not query provider registry for fails.example.com/awesomesauce/happycloud: Get http://placeholder-origin/fails-immediately/awesomesauce/happycloud/1.2.0/download/linux/amd64: EOF`,
+		},
+	}
+
+	// Sometimes error messages contain specific HTTP endpoint URLs, but
+	// since our test server is on a random port we'd not be able to
+	// consistently match those. Instead, we'll normalize the URLs.
+	urlPattern := regexp.MustCompile(`http://[^/]+/`)
+
+	cmpOpts := cmp.Comparer(Version.Same)
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s for %s_%s", test.provider, test.os, test.arch), func(t *testing.T) {
+			// TEMP: We don't yet have a function for parsing provider
+			// source addresses so we'll just fake it in here for now.
+			parts := strings.Split(test.provider, "/")
+			providerAddr := addrs.Provider{
+				Hostname:  svchost.Hostname(parts[0]),
+				Namespace: parts[1],
+				Type:      parts[2],
+			}
+
+			version := versions.MustParseVersion(test.version)
+
+			got, err := source.DownloadLocation(providerAddr, version, Platform{test.os, test.arch})
+
+			if err != nil {
+				if test.wantErr == "" {
+					t.Fatalf("wrong error\ngot:  %s\nwant: <nil>", err.Error())
+				}
+				gotErr := urlPattern.ReplaceAllLiteralString(err.Error(), "http://placeholder-origin/")
+				if got, want := gotErr, test.wantErr; got != want {
+					t.Fatalf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+				return
+			}
+
+			if test.wantErr != "" {
+				t.Fatalf("wrong error\ngot:  <nil>\nwant: %s", test.wantErr)
+			}
+
+			if diff := cmp.Diff(test.want, got, cmpOpts); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+	}
+
+}

internal/getproviders/types.go

@@ -0,0 +1,59 @@
+package getproviders
+
+import (
+	"crypto/sha256"
+	"runtime"
+
+	"github.com/apparentlymart/go-versions/versions"
+)
+
+// Version represents a particular single version of a provider.
+type Version = versions.Version
+
+// VersionList represents a list of versions. It is a []Version with some
+// extra methods for convenient filtering.
+type VersionList = versions.List
+
+// ParseVersion parses a "semver"-style version string into a Version value,
+// which is the version syntax we use for provider versions.
+func ParseVersion(str string) (Version, error) {
+	return versions.ParseVersion(str)
+}
+
+// Platform represents a target platform that a provider is or might be
+// available for.
+type Platform struct {
+	OS, Arch string
+}
+
+func (p Platform) String() string {
+	return p.OS + "_" + p.Arch
+}
+
+// CurrentPlatform is the platform where the current program is running.
+//
+// If attempting to install providers for use on the same system where the
+// installation process is running, this is the right platform to use.
+var CurrentPlatform = Platform{
+	OS:   runtime.GOOS,
+	Arch: runtime.GOARCH,
+}
+
+// PackageMeta represents the metadata related to a particular downloadable
+// provider package targeting a single platform.
+//
+// Package findproviders does no signature verification or protocol version
+// compatibility checking of its own. A caller receving a PackageMeta must
+// verify that it has a correct signature and supports a protocol version
+// accepted by the current version of Terraform before trying to use the
+// described package.
+type PackageMeta struct {
+	ProtocolVersions VersionList
+	TargetPlatform   Platform
+
+	Filename    string
+	DownloadURL string
+	SHA256Sum   [sha256.Size]byte
+
+	// TODO: Extra metadata for signature verification
+}