registry: adding provider functions to registry client

registry/client.go

@@ -20,10 +20,11 @@ import (
 )
 
 const (
-	xTerraformGet     = "X-Terraform-Get"
+	xTerraformGet      = "X-Terraform-Get"
-	xTerraformVersion = "X-Terraform-Version"
+	xTerraformVersion  = "X-Terraform-Version"
-	requestTimeout    = 10 * time.Second
+	requestTimeout     = 10 * time.Second
-	serviceID         = "modules.v1"
+	modulesServiceID   = "modules.v1"
+	providersServiceID = "providers.v1"
 )
 
 var tfVersion = version.String()
@@ -58,7 +59,7 @@ func NewClient(services *disco.Disco, client *http.Client) *Client {
 }
 
 // Discover qeuries the host, and returns the url for the registry.
-func (c *Client) Discover(host svchost.Hostname) *url.URL {
+func (c *Client) Discover(host svchost.Hostname, serviceID string) *url.URL {
 	service := c.services.DiscoverServiceURL(host, serviceID)
 	if service == nil {
 		return nil
@@ -76,7 +77,7 @@ func (c *Client) Versions(module *regsrc.Module) (*response.ModuleVersions, erro
 		return nil, err
 	}
 
-	service := c.Discover(host)
+	service := c.Discover(host, modulesServiceID)
 	if service == nil {
 		return nil, fmt.Errorf("host %s does not provide Terraform modules", host)
 	}
@@ -149,7 +150,7 @@ func (c *Client) Location(module *regsrc.Module, version string) (string, error)
 		return "", err
 	}
 
-	service := c.Discover(host)
+	service := c.Discover(host, modulesServiceID)
 	if service == nil {
 		return "", fmt.Errorf("host %s does not provide Terraform modules", host.ForDisplay())
 	}
@@ -225,3 +226,119 @@ func (c *Client) Location(module *regsrc.Module, version string) (string, error)
 
 	return location, nil
 }
+
+// TerraformProviderVersions queries the registry for a provider, and returns the available versions.
+func (c *Client) TerraformProviderVersions(provider *regsrc.TerraformProvider) (*response.TerraformProviderVersions, error) {
+	host, err := provider.SvcHost()
+	if err != nil {
+		return nil, err
+	}
+
+	service := c.Discover(host, providersServiceID)
+	if service == nil {
+		return nil, fmt.Errorf("host %s does not provide Terraform providers", host)
+	}
+
+	p, err := url.Parse(path.Join(provider.TerraformProvider(), "versions"))
+	if err != nil {
+		return nil, err
+	}
+
+	service = service.ResolveReference(p)
+
+	log.Printf("[DEBUG] fetching provider versions from %q", service)
+
+	req, err := http.NewRequest("GET", service.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	c.addRequestCreds(host, req)
+	req.Header.Set(xTerraformVersion, tfVersion)
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// OK
+	case http.StatusNotFound:
+		return nil, &errProviderNotFound{addr: provider}
+	default:
+		return nil, fmt.Errorf("error looking up provider versions: %s", resp.Status)
+	}
+
+	var versions response.TerraformProviderVersions
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&versions); err != nil {
+		return nil, err
+	}
+
+	return &versions, nil
+}
+
+// TerraformProviderLocation queries the registry for a provider download metadata
+func (c *Client) TerraformProviderLocation(provider *regsrc.TerraformProvider, version string) (*response.TerraformProviderPlatformLocation, error) {
+	host, err := provider.SvcHost()
+	if err != nil {
+		return nil, err
+	}
+
+	service := c.Discover(host, providersServiceID)
+	if service == nil {
+		return nil, fmt.Errorf("host %s does not provide Terraform providers", host.ForDisplay())
+	}
+
+	var p *url.URL
+	p, err = url.Parse(path.Join(
+		provider.TerraformProvider(),
+		version,
+		"download",
+		provider.OS,
+		provider.Arch,
+	))
+	if err != nil {
+		return nil, err
+	}
+
+	download := service.ResolveReference(p)
+
+	log.Printf("[DEBUG] looking up provider location from %q", download)
+
+	req, err := http.NewRequest("GET", download.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	c.addRequestCreds(host, req)
+	req.Header.Set(xTerraformVersion, tfVersion)
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var loc response.TerraformProviderPlatformLocation
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&loc); err != nil {
+		return nil, err
+	}
+
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusNoContent:
+		// OK
+	case http.StatusNotFound:
+		return nil, fmt.Errorf("provider %q version %q not found", provider.TerraformProvider(), version)
+	default:
+		// anything else is an error:
+		return nil, fmt.Errorf("error getting download location for %q: %s", provider.TerraformProvider(), resp.Status)
+	}
+
+	return &loc, nil
+}

registry/client_test.go

@@ -1,6 +1,7 @@
 package registry
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"testing"
@@ -199,3 +200,92 @@ func TestLookupLookupModuleError(t *testing.T) {
 		t.Fatal("error should not include the hostname. got:", err)
 	}
 }
+
+func TestLookupProviderVersions(t *testing.T) {
+	server := test.Registry()
+	defer server.Close()
+
+	client := NewClient(test.Disco(server), nil, nil)
+
+	tests := []struct {
+		name string
+	}{
+		{"foo"},
+		{"bar"},
+	}
+	for _, tt := range tests {
+		provider, err := regsrc.NewTerraformProvider(tt.name, "", "")
+		resp, err := client.TerraformProviderVersions(provider)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		name := fmt.Sprintf("terraform-providers/%s", tt.name)
+		if resp.ID != name {
+			t.Fatalf("expected provider name %q, got %q", name, resp.ID)
+		}
+
+		if len(resp.Versions) != 2 {
+			t.Fatal("expected 2 versions, got", len(resp.Versions))
+		}
+
+		for _, v := range resp.Versions {
+			_, err := version.NewVersion(v.Version)
+			if err != nil {
+				t.Fatalf("invalid version %q: %s", v, err)
+			}
+		}
+	}
+}
+
+func TestLookupProviderLocation(t *testing.T) {
+	server := test.Registry()
+	defer server.Close()
+
+	client := NewClient(test.Disco(server), nil, nil)
+
+	tests := []struct {
+		Name    string
+		Version string
+		Err     bool
+	}{
+		{
+			"foo",
+			"0.2.3",
+			false,
+		},
+		{
+			"bar",
+			"0.1.1",
+			false,
+		},
+		{
+			"baz",
+			"0.0.0",
+			true,
+		},
+	}
+	for _, tt := range tests {
+		// FIXME: the tests are set up to succeed - os/arch is not being validated at this time
+		p, err := regsrc.NewTerraformProvider(tt.Name, "linux", "amd64")
+		if err != nil {
+			t.Fatal(err)
+		}
+		locationMetadata, err := client.TerraformProviderLocation(p, tt.Version)
+		if tt.Err {
+			if err == nil {
+				t.Fatal("succeeded; want error")
+			}
+			return
+		} else if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		downloadURL := fmt.Sprintf("https://releases.hashicorp.com/terraform-provider-%s/%s/terraform-provider-%s.zip", tt.Name, tt.Version, tt.Name)
+
+		if locationMetadata.DownloadURL != downloadURL {
+			t.Fatalf("incorrect download URL: expected %q, got %q", downloadURL, locationMetadata.DownloadURL)
+		}
+	}
+
+}

registry/errors.go

@@ -21,3 +21,19 @@ func IsModuleNotFound(err error) bool {
 	_, ok := err.(*errModuleNotFound)
 	return ok
 }
+
+type errProviderNotFound struct {
+	addr *regsrc.TerraformProvider
+}
+
+func (e *errProviderNotFound) Error() string {
+	return fmt.Sprintf("provider %s not found", e.addr)
+}
+
+// IsProviderNotFound returns true only if the given error is a "provider not found"
+// error. This allows callers to recognize this particular error condition
+// as distinct from operational errors such as poor network connectivity.
+func IsProviderNotFound(err error) bool {
+	_, ok := err.(*errProviderNotFound)
+	return ok
+}

registry/regsrc/terraform_provider.go

@@ -0,0 +1,58 @@
+package regsrc
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/hashicorp/terraform/svchost"
+)
+
+var (
+	// DefaultProviderNamespace represents the namespace for canonical
+	// HashiCorp-controlled providers.
+	// REVIEWERS: Naming things is hard.
+	// * HashiCorpProviderNameSpace?
+	// * OfficialP...?
+	// * CanonicalP...?
+	DefaultProviderNamespace = "terraform-providers"
+)
+
+// TerraformProvider describes a Terraform Registry Provider source.
+type TerraformProvider struct {
+	RawHost      *FriendlyHost
+	RawNamespace string
+	RawName      string
+	OS           string
+	Arch         string
+}
+
+// NewTerraformProvider constructs a new provider source.
+func NewTerraformProvider(name, os, arch string) (*TerraformProvider, error) {
+	if os == "" {
+		os = runtime.GOOS
+	}
+	if arch == "" {
+		arch = runtime.GOARCH
+	}
+
+	p := &TerraformProvider{
+		RawHost:      PublicRegistryHost,
+		RawNamespace: DefaultProviderNamespace,
+		RawName:      name,
+		OS:           os,
+		Arch:         arch,
+	}
+
+	return p, nil
+}
+
+// Provider returns just the registry ID of the provider
+func (p *TerraformProvider) TerraformProvider() string {
+	return fmt.Sprintf("%s/%s", p.RawNamespace, p.RawName)
+}
+
+// SvcHost returns the svchost.Hostname for this provider. The
+// default PublicRegistryHost is returned.
+func (p *TerraformProvider) SvcHost() (svchost.Hostname, error) {
+	return svchost.ForComparison(PublicRegistryHost.Raw)
+}

registry/response/provider.go

@@ -0,0 +1,36 @@
+package response
+
+import (
+	"time"
+)
+
+// Provider is the response structure with the data for a single provider
+// version. This is just the metadata. A full provider response will be
+// ProviderDetail.
+type Provider struct {
+	ID string `json:"id"`
+
+	//---------------------------------------------------------------
+	// Metadata about the overall provider.
+
+	Owner       string    `json:"owner"`
+	Namespace   string    `json:"namespace"`
+	Name        string    `json:"name"`
+	Version     string    `json:"version"`
+	Description string    `json:"description"`
+	Source      string    `json:"source"`
+	PublishedAt time.Time `json:"published_at"`
+	Downloads   int       `json:"downloads"`
+}
+
+// ProviderDetail represents a Provider with full detail.
+type ProviderDetail struct {
+	Provider
+
+	//---------------------------------------------------------------
+	// The fields below are only set when requesting this specific
+	// module. They are available to easily know all available versions
+	// without multiple API calls.
+
+	Versions []string `json:"versions"` // All versions
+}

registry/response/provider_list.go

@@ -0,0 +1,7 @@
+package response
+
+// ProviderList is the response structure for a pageable list of providers.
+type ProviderList struct {
+	Meta      PaginationMeta `json:"meta"`
+	Providers []*Provider    `json:"providers"`
+}

registry/response/terraform_provider.go

@@ -0,0 +1,47 @@
+package response
+
+// TerraformProvider is the response structure for all required information for
+// Terraform to choose a download URL. It must include all versions and all
+// platforms for Terraform to perform version and os/arch constraint matching
+// locally.
+type TerraformProvider struct {
+	ID       string `json:"id"`
+	Verified bool   `json:"verified"`
+
+	Versions []*TerraformProviderVersion `json:"versions"`
+}
+
+// TerraformProviderVersion is the Terraform-specific response structure for a
+// provider version.
+type TerraformProviderVersion struct {
+	Version   string   `json:"version"`
+	Protocols []string `json:"protocols"`
+
+	Platforms []*TerraformProviderPlatform `json:"platforms"`
+}
+
+// TerraformProviderVersions is the Terraform-specific response structure for an
+// array of provider versions
+type TerraformProviderVersions struct {
+	ID       string                      `json:"id"`
+	Versions []*TerraformProviderVersion `json:"versions"`
+}
+
+// TerraformProviderPlatform is the Terraform-specific response structure for a
+// provider platform.
+type TerraformProviderPlatform struct {
+	OS   string `json:"os"`
+	Arch string `json:"arch"`
+}
+
+// TerraformProviderPlatformLocation is the Terraform-specific response
+// structure for a provider platform with all details required to perform a
+// download.
+type TerraformProviderPlatformLocation struct {
+	OS                  string `json:"os"`
+	Arch                string `json:"arch"`
+	Filename            string `json:"filename"`
+	DownloadURL         string `json:"download_url"`
+	ShasumsURL          string `json:"shasums_url"`
+	ShasumsSignatureURL string `json:"shasums_signature_url"`
+}

registry/test/mock_registry.go

@@ -25,7 +25,8 @@ func Disco(s *httptest.Server) *disco.Disco {
 	services := map[string]interface{}{
 		// Note that both with and without trailing slashes are supported behaviours
 		// TODO: add specific tests to enumerate both possibilities.
-		"modules.v1": fmt.Sprintf("%s/v1/modules", s.URL),
+		"modules.v1":   fmt.Sprintf("%s/v1/modules", s.URL),
+		"providers.v1": fmt.Sprintf("%s/v1/providers", s.URL),
 	}
 	d := disco.NewWithCredentialsSource(credsSrc)
 
@@ -43,6 +44,15 @@ type testMod struct {
 	version  string
 }
 
+// Map of provider names and location of test providers.
+// Only one version for now, as we only lookup latest from the registry.
+type testProvider struct {
+	version string
+	os      string
+	arch    string
+	url     string
+}
+
 const (
 	testCred = "test-auth-token"
 )
@@ -89,6 +99,23 @@ var testMods = map[string][]testMod{
 	},
 }
 
+var testProviders = map[string][]testProvider{
+	"terraform-providers/foo": {
+		{
+			version: "0.2.3",
+			url:     "https://releases.hashicorp.com/terraform-provider-foo/0.2.3/terraform-provider-foo.zip",
+		},
+		{version: "0.3.0"},
+	},
+	"terraform-providers/bar": {
+		{
+			version: "0.1.1",
+			url:     "https://releases.hashicorp.com/terraform-provider-bar/0.1.1/terraform-provider-bar.zip",
+		},
+		{version: "0.1.2"},
+	},
+}
+
 func latestVersion(versions []string) string {
 	var col version.Collection
 	for _, v := range versions {
@@ -106,7 +133,7 @@ func latestVersion(versions []string) string {
 func mockRegHandler() http.Handler {
 	mux := http.NewServeMux()
 
-	download := func(w http.ResponseWriter, r *http.Request) {
+	moduleDownload := func(w http.ResponseWriter, r *http.Request) {
 		p := strings.TrimLeft(r.URL.Path, "/")
 		// handle download request
 		re := regexp.MustCompile(`^([-a-z]+/\w+/\w+).*/download$`)
@@ -145,7 +172,7 @@ func mockRegHandler() http.Handler {
 		return
 	}
 
-	versions := func(w http.ResponseWriter, r *http.Request) {
+	moduleVersions := func(w http.ResponseWriter, r *http.Request) {
 		p := strings.TrimLeft(r.URL.Path, "/")
 		re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/versions$`)
 		matches := re.FindStringSubmatch(p)
@@ -197,12 +224,108 @@ func mockRegHandler() http.Handler {
 	mux.Handle("/v1/modules/",
 		http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if strings.HasSuffix(r.URL.Path, "/download") {
-				download(w, r)
+				moduleDownload(w, r)
 				return
 			}
 
 			if strings.HasSuffix(r.URL.Path, "/versions") {
-				versions(w, r)
+				moduleVersions(w, r)
+				return
+			}
+
+			http.NotFound(w, r)
+		})),
+	)
+
+	providerDownload := func(w http.ResponseWriter, r *http.Request) {
+		p := strings.TrimLeft(r.URL.Path, "/")
+		v := strings.Split(string(p), "/")
+
+		if len(v) != 6 {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		name := fmt.Sprintf("%s/%s", v[0], v[1])
+
+		providers, ok := testProviders[name]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+
+		// for this test / moment we will only return the one provider
+		loc := response.TerraformProviderPlatformLocation{
+			DownloadURL: providers[0].url,
+		}
+
+		js, err := json.Marshal(loc)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+
+	}
+
+	providerVersions := func(w http.ResponseWriter, r *http.Request) {
+		p := strings.TrimLeft(r.URL.Path, "/")
+		re := regexp.MustCompile(`^([-a-z]+/\w+)/versions$`)
+		matches := re.FindStringSubmatch(p)
+
+		if len(matches) != 2 {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		// check for auth
+		if strings.Contains(matches[1], "private/") {
+			if !strings.Contains(r.Header.Get("Authorization"), testCred) {
+				http.Error(w, "", http.StatusForbidden)
+			}
+		}
+
+		name := fmt.Sprintf("%s", matches[1])
+		versions, ok := testProviders[name]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+
+		// only adding the single requested provider for now
+		// this is the minimal that any regisry is epected to support
+		pvs := &response.TerraformProviderVersions{
+			ID: name,
+		}
+
+		for _, v := range versions {
+			pv := &response.TerraformProviderVersion{
+				Version: v.version,
+			}
+			pvs.Versions = append(pvs.Versions, pv)
+		}
+
+		js, err := json.Marshal(pvs)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+	}
+
+	mux.Handle("/v1/providers/",
+		http.StripPrefix("/v1/providers/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if strings.Contains(r.URL.Path, "/download") {
+				providerDownload(w, r)
+				return
+			}
+
+			if strings.HasSuffix(r.URL.Path, "/versions") {
+				providerVersions(w, r)
 				return
 			}
 
@@ -212,12 +335,12 @@ func mockRegHandler() http.Handler {
 
 	mux.HandleFunc("/.well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")
-		io.WriteString(w, `{"modules.v1":"http://localhost/v1/modules/"}`)
+		io.WriteString(w, `{"modules.v1":"http://localhost/v1/modules/", "providers.v1":"http://localhost/v1/providers/"}`)
 	})
 	return mux
 }
 
-// NewRegistry return an httptest server that mocks out some registry functionality.
+// Registry returns an httptest server that mocks out some registry functionality.
 func Registry() *httptest.Server {
 	return httptest.NewServer(mockRegHandler())
 }