Merge pull request #16773 from hashicorp/jbardin/registry

Registry client refactor
This commit is contained in:
James Bardin 2017-12-05 15:29:45 -05:00 committed by GitHub
commit 120709d0d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 275 additions and 263 deletions

View File

@ -1,16 +1,13 @@
package module
import (
"fmt"
"io/ioutil"
"log"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/disco"
)
@ -49,20 +46,3 @@ func testStorage(t *testing.T, d *disco.Disco) *Storage {
t.Helper()
return NewStorage(tempDir(t), d, nil)
}
// test discovery maps registry.terraform.io, localhost, localhost.localdomain,
// and example.com to the test server.
func testDisco(s *httptest.Server) *disco.Disco {
services := map[string]interface{}{
// Note that both with and without trailing slashes are supported behaviours
// TODO: add specific tests to enumerate both possibilities.
"modules.v1": fmt.Sprintf("%s/v1/modules", s.URL),
}
d := disco.NewDisco()
d.ForceHostServices(svchost.Hostname("registry.terraform.io"), services)
d.ForceHostServices(svchost.Hostname("localhost"), services)
d.ForceHostServices(svchost.Hostname("localhost.localdomain"), services)
d.ForceHostServices(svchost.Hostname("example.com"), services)
return d
}

View File

@ -9,6 +9,7 @@ import (
"path/filepath"
getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/registry"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
@ -73,20 +74,17 @@ type Storage struct {
Ui cli.Ui
// Mode is the GetMode that will be used for various operations.
Mode GetMode
registry *registry.Client
}
func NewStorage(dir string, services *disco.Disco, creds auth.CredentialsSource) *Storage {
s := &Storage{
StorageDir: dir,
Services: services,
Creds: creds,
}
regClient := registry.NewClient(services, creds, nil)
// make sure this isn't nil
if s.Services == nil {
s.Services = disco.NewDisco()
return &Storage{
StorageDir: dir,
registry: regClient,
}
return s
}
// loadManifest returns the moduleManifest file from the parent directory.
@ -318,7 +316,7 @@ func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, e
// we need to lookup available versions
// Only on Get if it's not found, on unconditionally on Update
if (s.Mode == GetModeGet && !found) || (s.Mode == GetModeUpdate) {
resp, err := s.lookupModuleVersions(mod)
resp, err := s.registry.Versions(mod)
if err != nil {
return rec, err
}
@ -338,7 +336,7 @@ func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, e
rec.Version = match.Version
rec.url, err = s.lookupModuleLocation(mod, rec.Version)
rec.url, err = s.registry.Location(mod, rec.Version)
if err != nil {
return rec, err
}

View File

@ -2,15 +2,20 @@ package module
import (
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/test"
)
func TestGetModule(t *testing.T) {
server := mockRegistry()
server := test.Registry()
defer server.Close()
disco := testDisco(server)
disco := test.Disco(server)
td, err := ioutil.TempDir("", "tf")
if err != nil {
@ -19,7 +24,7 @@ func TestGetModule(t *testing.T) {
defer os.RemoveAll(td)
storage := NewStorage(td, disco, nil)
// this module exists in a test fixture, and is known by the mockRegistry
// this module exists in a test fixture, and is known by the test.Registry
// relative to our cwd.
err = storage.GetModule(filepath.Join(td, "foo"), "registry/local/sub")
if err != nil {
@ -45,5 +50,140 @@ func TestGetModule(t *testing.T) {
if err != nil {
t.Fatal(err)
}
}
// GitHub archives always contain the module source in a single subdirectory,
// so the registry will return a path with with a `//*` suffix. We need to make
// sure this doesn't intefere with our internal handling of `//` subdir.
func TestRegistryGitHubArchive(t *testing.T) {
server := test.Registry()
defer server.Close()
disco := test.Disco(server)
storage := testStorage(t, disco)
tree := NewTree("", testConfig(t, "registry-tar-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
// stop the registry server, and make sure that we don't need to call out again
server.Close()
tree = NewTree("", testConfig(t, "registry-tar-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
actual := strings.TrimSpace(tree.String())
expected := strings.TrimSpace(treeLoadSubdirStr)
if actual != expected {
t.Fatalf("got: \n\n%s\nexpected: \n\n%s", actual, expected)
}
}
// Test that the //subdir notation can be used with registry modules
func TestRegisryModuleSubdir(t *testing.T) {
server := test.Registry()
defer server.Close()
disco := test.Disco(server)
storage := testStorage(t, disco)
tree := NewTree("", testConfig(t, "registry-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
actual := strings.TrimSpace(tree.String())
expected := strings.TrimSpace(treeLoadRegistrySubdirStr)
if actual != expected {
t.Fatalf("got: \n\n%s\nexpected: \n\n%s", actual, expected)
}
}
func TestAccRegistryDiscover(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("skipping ACC test")
}
// simply check that we get a valid github URL for this from the registry
module, err := regsrc.ParseModuleSource("hashicorp/consul/aws")
if err != nil {
t.Fatal(err)
}
s := NewStorage("/tmp", nil, nil)
loc, err := s.registry.Location(module, "")
if err != nil {
t.Fatal(err)
}
u, err := url.Parse(loc)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(u.Host, "github.com") {
t.Fatalf("expected host 'github.com', got: %q", u.Host)
}
if !strings.Contains(u.String(), "consul") {
t.Fatalf("url doesn't contain 'consul': %s", u.String())
}
}
func TestAccRegistryLoad(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("skipping ACC test")
}
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "registry-load"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
// TODO expand this further by fetching some metadata from the registry
actual := strings.TrimSpace(tree.String())
if !strings.Contains(actual, "(path: vault)") {
t.Fatal("missing vault module, got:\n", actual)
}
}

View File

@ -1,4 +1,4 @@
package module
package registry
import (
"encoding/json"
@ -12,75 +12,75 @@ import (
"time"
cleanhttp "github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/hashicorp/terraform/version"
)
const (
defaultRegistry = "registry.terraform.io"
registryServiceID = "registry.v1"
xTerraformGet = "X-Terraform-Get"
xTerraformVersion = "X-Terraform-Version"
requestTimeout = 10 * time.Second
serviceID = "modules.v1"
)
var (
httpClient *http.Client
tfVersion = version.String()
)
var tfVersion = version.String()
func init() {
httpClient = cleanhttp.DefaultPooledClient()
httpClient.Timeout = requestTimeout
// Client provides methods to query Terraform Registries.
type Client struct {
// this is the client to be used for all requests.
client *http.Client
// services is a required *disco.Disco, which may have services and
// credentials pre-loaded.
services *disco.Disco
// Creds optionally provides credentials for communicating with service
// providers.
creds auth.CredentialsSource
}
type errModuleNotFound string
func (e errModuleNotFound) Error() string {
return `module "` + string(e) + `" not found`
}
func (s *Storage) discoverRegURL(host svchost.Hostname) *url.URL {
regURL := s.Services.DiscoverServiceURL(host, serviceID)
if regURL == nil {
return nil
func NewClient(services *disco.Disco, creds auth.CredentialsSource, client *http.Client) *Client {
if services == nil {
services = disco.NewDisco()
}
if !strings.HasSuffix(regURL.Path, "/") {
regURL.Path += "/"
services.SetCredentialsSource(creds)
if client == nil {
client = cleanhttp.DefaultPooledClient()
client.Timeout = requestTimeout
}
return regURL
}
services.Transport = client.Transport.(*http.Transport)
func (s *Storage) addRequestCreds(host svchost.Hostname, req *http.Request) {
if s.Creds == nil {
return
}
creds, err := s.Creds.ForHost(host)
if err != nil {
log.Printf("[WARNING] Failed to get credentials for %s: %s (ignoring)", host, err)
return
}
if creds != nil {
creds.PrepareRequest(req)
return &Client{
client: client,
services: services,
creds: creds,
}
}
// Lookup module versions in the registry.
func (s *Storage) lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) {
// Discover qeuries the host, and returns the url for the registry.
func (c *Client) Discover(host svchost.Hostname) *url.URL {
service := c.services.DiscoverServiceURL(host, serviceID)
if !strings.HasSuffix(service.Path, "/") {
service.Path += "/"
}
return service
}
// Versions queries the registry for a module, and returns the available versions.
func (c *Client) Versions(module *regsrc.Module) (*response.ModuleVersions, error) {
host, err := module.SvcHost()
if err != nil {
return nil, err
}
service := s.discoverRegURL(host)
service := c.Discover(host)
if service == nil {
return nil, fmt.Errorf("host %s does not provide Terraform modules", host)
}
@ -99,10 +99,10 @@ func (s *Storage) lookupModuleVersions(module *regsrc.Module) (*response.ModuleV
return nil, err
}
s.addRequestCreds(host, req)
c.addRequestCreds(host, req)
req.Header.Set(xTerraformVersion, tfVersion)
resp, err := httpClient.Do(req)
resp, err := c.client.Do(req)
if err != nil {
return nil, err
}
@ -112,7 +112,7 @@ func (s *Storage) lookupModuleVersions(module *regsrc.Module) (*response.ModuleV
case http.StatusOK:
// OK
case http.StatusNotFound:
return nil, errModuleNotFound(module.String())
return nil, fmt.Errorf("module %q not found", module.String())
default:
return nil, fmt.Errorf("error looking up module versions: %s", resp.Status)
}
@ -133,14 +133,31 @@ func (s *Storage) lookupModuleVersions(module *regsrc.Module) (*response.ModuleV
return &versions, nil
}
// lookup the location of a specific module version in the registry
func (s *Storage) lookupModuleLocation(module *regsrc.Module, version string) (string, error) {
func (c *Client) addRequestCreds(host svchost.Hostname, req *http.Request) {
if c.creds == nil {
return
}
creds, err := c.creds.ForHost(host)
if err != nil {
log.Printf("[WARNING] Failed to get credentials for %s: %s (ignoring)", host, err)
return
}
if creds != nil {
creds.PrepareRequest(req)
}
}
// Location find the download location for a specific version module.
// This returns a string, because the final location may contain special go-getter syntax.
func (c *Client) Location(module *regsrc.Module, version string) (string, error) {
host, err := module.SvcHost()
if err != nil {
return "", err
}
service := s.discoverRegURL(host)
service := c.Discover(host)
if service == nil {
return "", fmt.Errorf("host %s does not provide Terraform modules", host.ForDisplay())
}
@ -163,10 +180,10 @@ func (s *Storage) lookupModuleLocation(module *regsrc.Module, version string) (s
return "", err
}
s.addRequestCreds(host, req)
c.addRequestCreds(host, req)
req.Header.Set(xTerraformVersion, tfVersion)
resp, err := httpClient.Do(req)
resp, err := c.client.Do(req)
if err != nil {
return "", err
}

View File

@ -1,4 +1,4 @@
package module
package registry
import (
"os"
@ -7,16 +7,15 @@ import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/registry/test"
"github.com/hashicorp/terraform/svchost/disco"
)
func TestLookupModuleVersions(t *testing.T) {
server := mockRegistry()
server := test.Registry()
defer server.Close()
regDisco := testDisco(server)
client := NewClient(test.Disco(server), nil, nil)
// test with and without a hostname
for _, src := range []string{
@ -28,8 +27,7 @@ func TestLookupModuleVersions(t *testing.T) {
t.Fatal(err)
}
s := &Storage{Services: regDisco}
resp, err := s.lookupModuleVersions(modsrc)
resp, err := client.Versions(modsrc)
if err != nil {
t.Fatal(err)
}
@ -58,11 +56,10 @@ func TestLookupModuleVersions(t *testing.T) {
}
func TestRegistryAuth(t *testing.T) {
server := mockRegistry()
server := test.Registry()
defer server.Close()
regDisco := testDisco(server)
storage := testStorage(t, regDisco)
client := NewClient(test.Disco(server), nil, nil)
src := "private/name/provider"
mod, err := regsrc.ParseModuleSource(src)
@ -71,36 +68,32 @@ func TestRegistryAuth(t *testing.T) {
}
// both should fail without auth
_, err = storage.lookupModuleVersions(mod)
_, err = client.Versions(mod)
if err == nil {
t.Fatal("expected error")
}
_, err = storage.lookupModuleLocation(mod, "1.0.0")
_, err = client.Location(mod, "1.0.0")
if err == nil {
t.Fatal("expected error")
}
storage.Creds = auth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{
svchost.Hostname(defaultRegistry): {"token": testCredentials},
})
client = NewClient(test.Disco(server), test.Credentials, nil)
_, err = storage.lookupModuleVersions(mod)
_, err = client.Versions(mod)
if err != nil {
t.Fatal(err)
}
_, err = storage.lookupModuleLocation(mod, "1.0.0")
_, err = client.Location(mod, "1.0.0")
if err != nil {
t.Fatal(err)
}
}
func TestLookupModuleLocationRelative(t *testing.T) {
server := mockRegistry()
server := test.Registry()
defer server.Close()
regDisco := testDisco(server)
storage := testStorage(t, regDisco)
client := NewClient(test.Disco(server), nil, nil)
src := "relative/foo/bar"
mod, err := regsrc.ParseModuleSource(src)
@ -108,7 +101,7 @@ func TestLookupModuleLocationRelative(t *testing.T) {
t.Fatal(err)
}
got, err := storage.lookupModuleLocation(mod, "0.2.0")
got, err := client.Location(mod, "0.2.0")
if err != nil {
t.Fatal(err)
}
@ -117,7 +110,6 @@ func TestLookupModuleLocationRelative(t *testing.T) {
if got != want {
t.Errorf("wrong location %s; want %s", got, want)
}
}
func TestAccLookupModuleVersions(t *testing.T) {
@ -129,17 +121,15 @@ func TestAccLookupModuleVersions(t *testing.T) {
// test with and without a hostname
for _, src := range []string{
"terraform-aws-modules/vpc/aws",
defaultRegistry + "/terraform-aws-modules/vpc/aws",
regsrc.PublicRegistryHost.String() + "/terraform-aws-modules/vpc/aws",
} {
modsrc, err := regsrc.ParseModuleSource(src)
if err != nil {
t.Fatal(err)
}
s := &Storage{
Services: regDisco,
}
resp, err := s.lookupModuleVersions(modsrc)
s := NewClient(regDisco, nil, nil)
resp, err := s.Versions(modsrc)
if err != nil {
t.Fatal(err)
}
@ -169,11 +159,10 @@ func TestAccLookupModuleVersions(t *testing.T) {
// the error should reference the config source exatly, not the discovered path.
func TestLookupLookupModuleError(t *testing.T) {
server := mockRegistry()
server := test.Registry()
defer server.Close()
regDisco := testDisco(server)
storage := testStorage(t, regDisco)
client := NewClient(test.Disco(server), nil, nil)
// this should not be found in teh registry
src := "bad/local/path"
@ -182,7 +171,7 @@ func TestLookupLookupModuleError(t *testing.T) {
t.Fatal(err)
}
_, err = storage.lookupModuleLocation(mod, "0.2.0")
_, err = client.Location(mod, "0.2.0")
if err == nil {
t.Fatal("expected error")
}

View File

@ -1,4 +1,4 @@
package module
package test
import (
"encoding/json"
@ -6,18 +6,36 @@ import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"regexp"
"sort"
"strings"
"testing"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
)
// Disco return a *disco.Disco mapping registry.terraform.io, localhost,
// localhost.localdomain, and example.com to the test server.
func Disco(s *httptest.Server) *disco.Disco {
services := map[string]interface{}{
// Note that both with and without trailing slashes are supported behaviours
// TODO: add specific tests to enumerate both possibilities.
"modules.v1": fmt.Sprintf("%s/v1/modules", s.URL),
}
d := disco.NewDisco()
d.ForceHostServices(svchost.Hostname("registry.terraform.io"), services)
d.ForceHostServices(svchost.Hostname("localhost"), services)
d.ForceHostServices(svchost.Hostname("localhost.localdomain"), services)
d.ForceHostServices(svchost.Hostname("example.com"), services)
return d
}
// Map of module names and location of test modules.
// Only one version for now, as we only lookup latest from the registry.
type testMod struct {
@ -26,7 +44,14 @@ type testMod struct {
}
const (
testCredentials = "test-auth-token"
testCred = "test-auth-token"
)
var (
regHost = svchost.Hostname(regsrc.PublicRegistryHost.Normalized())
Credentials = auth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{
regHost: {"token": testCred},
})
)
// All the locationes from the mockRegistry start with a file:// scheme. If
@ -94,8 +119,9 @@ func mockRegHandler() http.Handler {
// check for auth
if strings.Contains(matches[0], "private/") {
if !strings.Contains(r.Header.Get("Authorization"), testCredentials) {
if !strings.Contains(r.Header.Get("Authorization"), testCred) {
http.Error(w, "", http.StatusForbidden)
return
}
}
@ -130,7 +156,7 @@ func mockRegHandler() http.Handler {
// check for auth
if strings.Contains(matches[1], "private/") {
if !strings.Contains(r.Header.Get("Authorization"), testCredentials) {
if !strings.Contains(r.Header.Get("Authorization"), testCred) {
http.Error(w, "", http.StatusForbidden)
}
}
@ -191,145 +217,7 @@ func mockRegHandler() http.Handler {
return mux
}
// Just enough like a registry to exercise our code.
// Returns the location of the latest version
func mockRegistry() *httptest.Server {
server := httptest.NewServer(mockRegHandler())
return server
}
// GitHub archives always contain the module source in a single subdirectory,
// so the registry will return a path with with a `//*` suffix. We need to make
// sure this doesn't intefere with our internal handling of `//` subdir.
func TestRegistryGitHubArchive(t *testing.T) {
server := mockRegistry()
defer server.Close()
disco := testDisco(server)
storage := testStorage(t, disco)
tree := NewTree("", testConfig(t, "registry-tar-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
// stop the registry server, and make sure that we don't need to call out again
server.Close()
tree = NewTree("", testConfig(t, "registry-tar-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
actual := strings.TrimSpace(tree.String())
expected := strings.TrimSpace(treeLoadSubdirStr)
if actual != expected {
t.Fatalf("got: \n\n%s\nexpected: \n\n%s", actual, expected)
}
}
// Test that the //subdir notation can be used with registry modules
func TestRegisryModuleSubdir(t *testing.T) {
server := mockRegistry()
defer server.Close()
disco := testDisco(server)
storage := testStorage(t, disco)
tree := NewTree("", testConfig(t, "registry-subdir"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
actual := strings.TrimSpace(tree.String())
expected := strings.TrimSpace(treeLoadRegistrySubdirStr)
if actual != expected {
t.Fatalf("got: \n\n%s\nexpected: \n\n%s", actual, expected)
}
}
func TestAccRegistryDiscover(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("skipping ACC test")
}
// simply check that we get a valid github URL for this from the registry
module, err := regsrc.ParseModuleSource("hashicorp/consul/aws")
if err != nil {
t.Fatal(err)
}
s := NewStorage("/tmp", nil, nil)
loc, err := s.lookupModuleLocation(module, "")
if err != nil {
t.Fatal(err)
}
u, err := url.Parse(loc)
if err != nil {
t.Fatal(err)
}
if !strings.HasSuffix(u.Host, "github.com") {
t.Fatalf("expected host 'github.com', got: %q", u.Host)
}
if !strings.Contains(u.String(), "consul") {
t.Fatalf("url doesn't contain 'consul': %s", u.String())
}
}
func TestAccRegistryLoad(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip("skipping ACC test")
}
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "registry-load"))
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
if !tree.Loaded() {
t.Fatal("should be loaded")
}
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
// TODO expand this further by fetching some metadata from the registry
actual := strings.TrimSpace(tree.String())
if !strings.Contains(actual, "(path: vault)") {
t.Fatal("missing vault module, got:\n", actual)
}
// NewRegistry return an httptest server that mocks out some registry functionality.
func Registry() *httptest.Server {
return httptest.NewServer(mockRegHandler())
}