a series of test commits

wire up HTTP so we can test the mock discovery service

test lookupModuleVersions

Add a versions endpoint to the mock registry, and use that to verify the
lookupModuleVersions behavior.

lookupModuleVersions takes a Disco as the argument so a custom Transport
can be injected, and uses that transport for its own client if it set.

test looking up modules with default registry

Add registry.terrform.io to the hostname that the mock registry resolves
to localhost.

ACC test looking up module versions

Lookup a basic module for the available version in the default registry.

config/module/get_test.go

@@ -1,7 +1,9 @@
 package module
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -14,6 +16,7 @@ import (
 
 	getter "github.com/hashicorp/go-getter"
 	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform/registry/response"
 )
 
 // Map of module names and location of test modules.
@@ -26,22 +29,28 @@ type testMod struct {
 // All the locationes from the mockRegistry start with a file:// scheme. If
 // the the location string here doesn't have a scheme, the mockRegistry will
 // find the absolute path and return a complete URL.
-var testMods = map[string]testMod{
+var testMods = map[string][]testMod{
-	"registry/foo/bar": {
+	"registry/foo/bar": {{
 		location: "file:///download/registry/foo/bar/0.2.3//*?archive=tar.gz",
 		version:  "0.2.3",
-	},
+	}},
-	"registry/foo/baz": {
+	"registry/foo/baz": {{
 		location: "file:///download/registry/foo/baz/1.10.0//*?archive=tar.gz",
 		version:  "1.10.0",
-	},
+	}},
-	"registry/local/sub": {
+	"registry/local/sub": {{
 		location: "test-fixtures/registry-tar-subdir/foo.tgz//*?archive=tar.gz",
 		version:  "0.1.2",
-	},
+	}},
-	"exists-in-registry/identifier/provider": {
+	"exists-in-registry/identifier/provider": {{
 		location: "file:///registry/exists",
 		version:  "0.2.0",
+	}},
+	"test-versions/name/provider": {
+		{version: "2.2.0"},
+		{version: "2.1.1"},
+		{version: "1.2.2"},
+		{version: "1.2.1"},
 	},
 }
 
@@ -59,45 +68,114 @@ func latestVersion(versions []string) string {
 	return col[len(col)-1].String()
 }
 
-// Just enough like a registry to exercise our code.
+func mockRegHandler() http.Handler {
-// Returns the location of the latest version
-func mockRegistry() *httptest.Server {
 	mux := http.NewServeMux()
-	server := httptest.NewServer(mux)
+
+	download := func(w http.ResponseWriter, r *http.Request) {
+		p := strings.TrimLeft(r.URL.Path, "/")
+		// handle download request
+		re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/download$`)
+		// download lookup
+		matches := re.FindStringSubmatch(p)
+		if len(matches) != 2 {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		versions, ok := testMods[matches[1]]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		mod := versions[0]
+
+		location := mod.location
+		if !strings.HasPrefix(location, "file:///") {
+			// we can't use filepath.Abs because it will clean `//`
+			wd, _ := os.Getwd()
+			location = fmt.Sprintf("file://%s/%s", wd, location)
+		}
+
+		w.Header().Set("X-Terraform-Get", location)
+		w.WriteHeader(http.StatusNoContent)
+		// no body
+		return
+	}
+
+	versions := func(w http.ResponseWriter, r *http.Request) {
+		p := strings.TrimLeft(r.URL.Path, "/")
+		re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/versions$`)
+		matches := re.FindStringSubmatch(p)
+		if len(matches) != 2 {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		name := matches[1]
+		versions, ok := testMods[name]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+
+		// only adding the single requested module for now
+		// this is the minimal that any regisry is epected to support
+		mpvs := &response.ModuleProviderVersions{
+			Source: name,
+		}
+
+		for _, v := range versions {
+			mv := &response.ModuleVersion{
+				Version: v.version,
+			}
+			mpvs.Versions = append(mpvs.Versions, mv)
+		}
+
+		resp := response.ModuleVersions{
+			Modules: []*response.ModuleProviderVersions{mpvs},
+		}
+
+		js, err := json.Marshal(resp)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(js)
+	}
 
 	mux.Handle("/v1/modules/",
 		http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			p := strings.TrimLeft(r.URL.Path, "/")
+			if strings.HasSuffix(r.URL.Path, "/download") {
-			// handle download request
+				download(w, r)
-			download := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/download$`)
-
-			// download lookup
-			matches := download.FindStringSubmatch(p)
-			if len(matches) != 2 {
-				w.WriteHeader(http.StatusBadRequest)
 				return
 			}
 
-			mod, ok := testMods[matches[1]]
+			if strings.HasSuffix(r.URL.Path, "/versions") {
-			if !ok {
+				versions(w, r)
-				w.WriteHeader(http.StatusNotFound)
 				return
 			}
 
-			location := mod.location
+			http.NotFound(w, r)
-			if !strings.HasPrefix(location, "file:///") {
-				// we can't use filepath.Abs because it will clean `//`
-				wd, _ := os.Getwd()
-				location = fmt.Sprintf("file://%s/%s", wd, location)
-			}
-
-			w.Header().Set("X-Terraform-Get", location)
-			w.WriteHeader(http.StatusNoContent)
-			// no body
-			return
 		})),
 	)
 
+	mux.HandleFunc("/.well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		io.WriteString(w, `{"modules.v1":"/v1/modules/"}`)
+	})
+	return mux
+}
+
+// Just enough like a registry to exercise our code.
+// Returns the location of the latest version
+func mockRegistry() *httptest.Server {
+	server := httptest.NewServer(mockRegHandler())
+	return server
+}
+
+func mockTLSRegistry() *httptest.Server {
+	server := httptest.NewTLSServer(mockRegHandler())
 	return server
 }
 
@@ -138,12 +216,12 @@ func TestDetectRegistry(t *testing.T) {
 	}{
 		{
 			source:   "registry/foo/bar",
-			location: testMods["registry/foo/bar"].location,
+			location: testMods["registry/foo/bar"][0].location,
 			found:    true,
 		},
 		{
 			source:   "registry/foo/baz",
-			location: testMods["registry/foo/baz"].location,
+			location: testMods["registry/foo/baz"][0].location,
 			found:    true,
 		},
 		// this should not be found, and is no longer valid as a local source

config/module/registry.go

@@ -30,7 +30,6 @@ const (
 var (
 	client    *http.Client
 	tfVersion = version.String()
-	regDisco  = disco.NewDisco()
 )
 
 func init() {
@@ -45,21 +44,27 @@ func (e errModuleNotFound) Error() string {
 }
 
 // Lookup module versions in the registry.
-func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) {
+func lookupModuleVersions(regDisco *disco.Disco, module *regsrc.Module) (*response.ModuleVersions, error) {
 	if module.RawHost == nil {
 		module.RawHost = regsrc.NewFriendlyHost(defaultRegistry)
 	}
 
-	regUrl := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID)
+	regURL := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID)
-	if regUrl == nil {
+	if regURL == nil {
-		regUrl = &url.URL{
+		regURL = &url.URL{
 			Scheme: "https",
 			Host:   module.RawHost.String(),
 			Path:   defaultApiPath,
 		}
 	}
 
-	location := fmt.Sprintf("%s/%s/%s/%s/versions", regUrl, module.RawNamespace, module.RawName, module.RawProvider)
+	service := regURL.String()
+
+	if service[len(service)-1] != '/' {
+		service += "/"
+	}
+
+	location := fmt.Sprintf("%s%s/%s/%s/versions", service, module.RawNamespace, module.RawName, module.RawProvider)
 	log.Printf("[DEBUG] fetching module versions from %q", location)
 
 	req, err := http.NewRequest("GET", location, nil)
@@ -69,6 +74,15 @@ func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, erro
 
 	req.Header.Set(xTerraformVersion, tfVersion)
 
+	// if discovery required a custom transport, then we should use that too
+	client := client
+	if regDisco.Transport != nil {
+		client = &http.Client{
+			Transport: regDisco.Transport,
+			Timeout:   requestTimeout,
+		}
+	}
+
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err

config/module/registry_test.go

@@ -0,0 +1,152 @@
+package module
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	cleanhttp "github.com/hashicorp/go-cleanhttp"
+	version "github.com/hashicorp/go-version"
+	"github.com/hashicorp/terraform/registry/regsrc"
+	"github.com/hashicorp/terraform/svchost/disco"
+)
+
+// Return a transport to use for this test server.
+// This not only loads the tls.Config from the test server for proper cert
+// validation, but also inserts a Dialer that resolves localhost and
+// example.com to 127.0.0.1 with the correct port, since 127.0.0.1 on its own
+// isn't a valid registry hostname.
+// TODO: cert validation not working here, so we use don't verify for now.
+func mockTransport(server *httptest.Server) *http.Transport {
+	u, _ := url.Parse(server.URL)
+	_, port, _ := net.SplitHostPort(u.Host)
+
+	transport := cleanhttp.DefaultTransport()
+	transport.TLSClientConfig = server.TLS
+	transport.TLSClientConfig.InsecureSkipVerify = true
+	transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, _, _ := net.SplitHostPort(addr)
+		switch host {
+		case "example.com", "localhost", "localhost.localdomain", "registry.terraform.io":
+			addr = "127.0.0.1"
+			if port != "" {
+				addr += ":" + port
+			}
+		}
+		return (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext(ctx, network, addr)
+	}
+	return transport
+}
+
+func TestMockDiscovery(t *testing.T) {
+	server := mockTLSRegistry()
+	defer server.Close()
+
+	regDisco := disco.NewDisco()
+	regDisco.Transport = mockTransport(server)
+
+	regURL := regDisco.DiscoverServiceURL("example.com", serviceID)
+
+	if regURL == nil {
+		t.Fatal("no registry service discovered")
+	}
+
+	if regURL.Host != "example.com" {
+		t.Fatal("expected registry host example.com, got:", regURL.Host)
+	}
+}
+
+func TestLookupModuleVersions(t *testing.T) {
+	server := mockTLSRegistry()
+	defer server.Close()
+	regDisco := disco.NewDisco()
+	regDisco.Transport = mockTransport(server)
+
+	// test with and without a hostname
+	for _, src := range []string{
+		"example.com/test-versions/name/provider",
+		"test-versions/name/provider",
+	} {
+		modsrc, err := regsrc.ParseModuleSource(src)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		resp, err := lookupModuleVersions(regDisco, modsrc)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(resp.Modules) != 1 {
+			t.Fatal("expected 1 module, got", len(resp.Modules))
+		}
+
+		mod := resp.Modules[0]
+		name := "test-versions/name/provider"
+		if mod.Source != name {
+			t.Fatalf("expected module name %q, got %q", name, mod.Source)
+		}
+
+		if len(mod.Versions) != 4 {
+			t.Fatal("expected 4 versions, got", len(mod.Versions))
+		}
+
+		for _, v := range mod.Versions {
+			_, err := version.NewVersion(v.Version)
+			if err != nil {
+				t.Fatalf("invalid version %q: %s", v.Version, err)
+			}
+		}
+	}
+}
+
+func TestACCLookupModuleVersions(t *testing.T) {
+	server := mockTLSRegistry()
+	defer server.Close()
+	regDisco := disco.NewDisco()
+
+	// test with and without a hostname
+	for _, src := range []string{
+		"terraform-aws-modules/vpc/aws",
+		defaultRegistry + "/terraform-aws-modules/vpc/aws",
+	} {
+		modsrc, err := regsrc.ParseModuleSource(src)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		resp, err := lookupModuleVersions(regDisco, modsrc)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if len(resp.Modules) != 1 {
+			t.Fatal("expected 1 module, got", len(resp.Modules))
+		}
+
+		mod := resp.Modules[0]
+		name := "terraform-aws-modules/vpc/aws"
+		if mod.Source != name {
+			t.Fatalf("expected module name %q, got %q", name, mod.Source)
+		}
+
+		if len(mod.Versions) == 0 {
+			t.Fatal("expected multiple versions, got 0")
+		}
+
+		for _, v := range mod.Versions {
+			_, err := version.NewVersion(v.Version)
+			if err != nil {
+				t.Fatalf("invalid version %q: %s", v.Version, err)
+			}
+		}
+	}
+}