Use the new regsrc and response packages

Adds basic detector for registry module source strings. While this isn't
a thorough validation, this will eliminate anything that is definitely
not a registry module, and split out our host and module id strings.

lookupModuleVersions interrogates the registry for the available
versions of a particular module and the tree of dependencies.

config/module/detector_test.go

@@ -0,0 +1,112 @@
+package module
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/terraform/registry/regsrc"
+)
+
+func TestParseRegistrySource(t *testing.T) {
+	for _, tc := range []struct {
+		source      string
+		host        string
+		id          string
+		err         bool
+		notRegistry bool
+	}{
+		{ // simple source id
+			source: "namespace/id/provider",
+			id:     "namespace/id/provider",
+		},
+		{ // source with hostname
+			source: "registry.com/namespace/id/provider",
+			host:   "registry.com",
+			id:     "namespace/id/provider",
+		},
+		{ // source with hostname and port
+			source: "registry.com:4443/namespace/id/provider",
+			host:   "registry.com:4443",
+			id:     "namespace/id/provider",
+		},
+		{ // too many parts
+			source:      "registry.com/namespace/id/provider/extra",
+			notRegistry: true,
+		},
+		{ // local path
+			source:      "./local/file/path",
+			notRegistry: true,
+		},
+		{ // local path with hostname
+			source:      "./registry.com/namespace/id/provider",
+			notRegistry: true,
+		},
+		{ // full URL
+			source:      "https://example.com/foo/bar/baz",
+			notRegistry: true,
+		},
+		{ // punycode host not allowed in source
+			source: "xn--80akhbyknj4f.com/namespace/id/provider",
+			err:    true,
+		},
+		{ // simple source id with subdir
+			source: "namespace/id/provider//subdir",
+			id:     "namespace/id/provider",
+		},
+		{ // source with hostname and subdir
+			source: "registry.com/namespace/id/provider//subdir",
+			host:   "registry.com",
+			id:     "namespace/id/provider",
+		},
+		{ // source with hostname
+			source: "registry.com/namespace/id/provider",
+			host:   "registry.com",
+			id:     "namespace/id/provider",
+		},
+		{ // we special case github
+			source:      "github.com/namespace/id/provider",
+			notRegistry: true,
+		},
+		{ // we special case github ssh
+			source:      "git@github.com:namespace/id/provider",
+			notRegistry: true,
+		},
+		{ // we special case bitbucket
+			source:      "bitbucket.org/namespace/id/provider",
+			notRegistry: true,
+		},
+	} {
+		t.Run(tc.source, func(t *testing.T) {
+			mod, err := regsrc.ParseModuleSource(tc.source)
+			if tc.notRegistry {
+				if err != regsrc.ErrInvalidModuleSource {
+					t.Fatalf("%q should not be a registry source, got err %v", tc.source, err)
+				}
+				return
+			}
+
+			if tc.err {
+				if err == nil {
+					t.Fatal("expected error")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			id := fmt.Sprintf("%s/%s/%s", mod.RawNamespace, mod.RawName, mod.RawProvider)
+
+			if tc.host != "" {
+				if mod.RawHost.Normalized() != tc.host {
+					t.Fatalf("expected host %q, got %q", tc.host, mod.RawHost)
+				}
+			}
+
+			if tc.id != id {
+				t.Fatalf("expected id %q, got %q", tc.id, id)
+			}
+		})
+	}
+}

config/module/get.go

@@ -65,8 +65,7 @@ func GetCopy(dst, src string) error {
 }
 
 const (
-	registryAPI   = "https://registry.terraform.io/v1/modules"
-	xTerraformGet = "X-Terraform-Get"
+	registryAPI = "https://registry.terraform.io/v1/modules"
 )
 
 var detectors = []getter.Detector{
@@ -79,7 +78,7 @@ var detectors = []getter.Detector{
 
 // these prefixes can't be registry IDs
 // "http", "../", "./", "/", "getter::", etc
-var skipRegistry = regexp.MustCompile(`^(http|[.]{1,2}/|/|[A-Za-z0-9]+::)`).MatchString
+var oldSkipRegistry = regexp.MustCompile(`^(http|[.]{1,2}/|/|[A-Za-z0-9]+::)`).MatchString
 
 // registryDetector implements getter.Detector to detect Terraform Registry modules.
 // If a path looks like a registry module identifier, attempt to locate it in
@@ -95,7 +94,7 @@ type registryDetector struct {
 func (d registryDetector) Detect(src, _ string) (string, bool, error) {
 	// the namespace can't start with "http", a relative or absolute path, or
 	// contain a go-getter "forced getter"
-	if skipRegistry(src) {
+	if oldSkipRegistry(src) {
 		return "", false, nil
 	}
 

config/module/get_test.go

@@ -91,7 +91,7 @@ func mockRegistry() *httptest.Server {
 				location = fmt.Sprintf("file://%s/%s", wd, location)
 			}
 
-			w.Header().Set(xTerraformGet, location)
+			w.Header().Set("X-Terraform-Get", location)
 			w.WriteHeader(http.StatusNoContent)
 			// no body
 			return

config/module/registry.go

@@ -0,0 +1,95 @@
+package module
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+
+	cleanhttp "github.com/hashicorp/go-cleanhttp"
+
+	"github.com/hashicorp/terraform/registry/regsrc"
+	"github.com/hashicorp/terraform/registry/response"
+	"github.com/hashicorp/terraform/svchost"
+	"github.com/hashicorp/terraform/svchost/disco"
+	"github.com/hashicorp/terraform/version"
+)
+
+const (
+	defaultRegistry   = "registry.terraform.io"
+	defaultApiPath    = "/v1/modules"
+	registryServiceID = "registry.v1"
+	xTerraformGet     = "X-Terraform-Get"
+	xTerraformVersion = "X-Terraform-Version"
+	requestTimeout    = 10 * time.Second
+	serviceID         = "modules.v1"
+)
+
+var (
+	client    *http.Client
+	tfVersion = version.String()
+	regDisco  = disco.NewDisco()
+)
+
+func init() {
+	client = cleanhttp.DefaultPooledClient()
+	client.Timeout = requestTimeout
+}
+
+type errModuleNotFound string
+
+func (e errModuleNotFound) Error() string {
+	return `module "` + string(e) + `" not found`
+}
+
+// Lookup module versions in the registry.
+func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) {
+	if module.RawHost == nil {
+		module.RawHost = regsrc.NewFriendlyHost(defaultRegistry)
+	}
+
+	regUrl := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID)
+	if regUrl == nil {
+		regUrl = &url.URL{
+			Scheme: "https",
+			Host:   module.RawHost.String(),
+			Path:   defaultApiPath,
+		}
+	}
+
+	location := fmt.Sprintf("%s/%s/%s/%s/versions", regUrl, module.RawNamespace, module.RawName, module.RawProvider)
+	log.Printf("[DEBUG] fetching module versions from %q", location)
+
+	req, err := http.NewRequest("GET", location, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set(xTerraformVersion, tfVersion)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		// OK
+	case http.StatusNotFound:
+		return nil, errModuleNotFound(module.String())
+	default:
+		return nil, fmt.Errorf("error looking up module versions: %s", resp.Status)
+	}
+
+	var versions response.ModuleVersions
+
+	dec := json.NewDecoder(resp.Body)
+	if err := dec.Decode(&versions); err != nil {
+		return nil, err
+	}
+
+	return &versions, nil
+}

config/module/versions.go

@@ -0,0 +1,95 @@
+package module
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform/registry/response"
+)
+
+const anyVersion = ">=0.0.0"
+
+// return the newest version that satisfies the provided constraint
+func newest(versions []string, constraint string) (string, error) {
+	if constraint == "" {
+		constraint = anyVersion
+	}
+	cs, err := version.NewConstraint(constraint)
+	if err != nil {
+		return "", err
+	}
+
+	switch len(versions) {
+	case 0:
+		return "", errors.New("no versions found")
+	case 1:
+		v, err := version.NewVersion(versions[0])
+		if err != nil {
+			return "", err
+		}
+
+		if !cs.Check(v) {
+			return "", fmt.Errorf("no version found matching constraint %q", constraint)
+		}
+		return versions[0], nil
+	}
+
+	sort.Slice(versions, func(i, j int) bool {
+		// versions should have already been validated
+		// sort invalid version strings to the end
+		iv, err := version.NewVersion(versions[i])
+		if err != nil {
+			return true
+		}
+		jv, err := version.NewVersion(versions[j])
+		if err != nil {
+			return true
+		}
+		return iv.GreaterThan(jv)
+	})
+
+	// versions are now in order, so just find the first which satisfies the
+	// constraint
+	for i := range versions {
+		v, err := version.NewVersion(versions[i])
+		if err != nil {
+			continue
+		}
+		if cs.Check(v) {
+			return versions[i], nil
+		}
+	}
+
+	return "", nil
+}
+
+// return the newest *moduleVersion that matches the given constraint
+// TODO: reconcile these two types and newest* functions
+func newestVersion(moduleVersions []*response.ModuleVersion, constraint string) (*response.ModuleVersion, error) {
+	var versions []string
+	modules := make(map[string]*response.ModuleVersion)
+
+	for _, m := range moduleVersions {
+		versions = append(versions, m.Version)
+		modules[m.Version] = m
+	}
+
+	match, err := newest(versions, constraint)
+	return modules[match], err
+}
+
+// return the newest moduleRecord that matches the given constraint
+func newestRecord(moduleVersions []moduleRecord, constraint string) (moduleRecord, error) {
+	var versions []string
+	modules := make(map[string]moduleRecord)
+
+	for _, m := range moduleVersions {
+		versions = append(versions, m.Version)
+		modules[m.Version] = m
+	}
+
+	match, err := newest(versions, constraint)
+	return modules[match], err
+}

config/module/versions_test.go

@@ -0,0 +1,60 @@
+package module
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/registry/response"
+)
+
+func TestNewestModuleVersion(t *testing.T) {
+	mpv := &response.ModuleProviderVersions{
+		Source: "registry/test/module",
+		Versions: []*response.ModuleVersion{
+			{Version: "0.0.4"},
+			{Version: "0.3.1"},
+			{Version: "2.0.1"},
+			{Version: "1.2.0"},
+		},
+	}
+
+	m, err := newestVersion(mpv.Versions, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := "2.0.1"
+	if m.Version != expected {
+		t.Fatalf("expected version %q, got %q", expected, m.Version)
+	}
+
+	// now with a constraint
+	m, err = newestVersion(mpv.Versions, "~>1.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected = "1.2.0"
+	if m.Version != expected {
+		t.Fatalf("expected version %q, got %q", expected, m.Version)
+	}
+}
+
+func TestNewestInvalidModuleVersion(t *testing.T) {
+	mpv := &response.ModuleProviderVersions{
+		Source: "registry/test/module",
+		Versions: []*response.ModuleVersion{
+			{Version: "WTF"},
+			{Version: "2.0.1"},
+		},
+	}
+
+	m, err := newestVersion(mpv.Versions, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := "2.0.1"
+	if m.Version != expected {
+		t.Fatalf("expected version %q, got %q", expected, m.Version)
+	}
+}