ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #16481 from hashicorp/jbardin/registry-auth 

registry auth
This commit is contained in:
James Bardin 2017-10-27 17:36:41 -04:00 committed by GitHub
parent 76b9cac1a4 ac68f9a16b
commit 9df2389ba2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 331 additions and 319 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
command/command_test.go
View File

@ -19,7 +19,6 @@ import (
"syscall"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/helper/logging"
"github.com/hashicorp/terraform/terraform"
@ -108,8 +107,9 @@ func testModule(t *testing.T, name string) *module.Tree {
t.Fatalf("err: %s", err)
}
s := &getter.FolderStorage{StorageDir: tempDir(t)}
if err := mod.Load(s, module.GetModeGet); err != nil {
s := module.NewStorage(tempDir(t), nil, nil)
s.Mode = module.GetModeGet
if err := mod.Load(s); err != nil {
t.Fatalf("err: %s", err)
}

2
command/get.go
View File

@ -81,7 +81,7 @@ func getModules(m *Meta, path string, mode module.GetMode) error {
return fmt.Errorf("Error loading configuration: %s", err)
}
err = mod.Load(m.moduleStorage(m.DataDir()), mode)
err = mod.Load(m.moduleStorage(m.DataDir(), mode))
if err != nil {
return fmt.Errorf("Error loading modules: %s", err)
}

14
command/meta.go
View File

@ -16,11 +16,11 @@ import (
"strings"
"time"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/backend"
"github.com/hashicorp/terraform/backend/local"
"github.com/hashicorp/terraform/command/format"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/helper/experiment"
"github.com/hashicorp/terraform/helper/variables"
"github.com/hashicorp/terraform/helper/wrappedstreams"
@ -368,13 +368,11 @@ func (m *Meta) flagSet(n string) *flag.FlagSet {
// moduleStorage returns the module.Storage implementation used to store
// modules for commands.
func (m *Meta) moduleStorage(root string) getter.Storage {
return &uiModuleStorage{
Storage: &getter.FolderStorage{
StorageDir: filepath.Join(root, "modules"),
},
Ui: m.Ui,
}
func (m *Meta) moduleStorage(root string, mode module.GetMode) *module.Storage {
s := module.NewStorage(filepath.Join(root, "modules"), m.Services, m.Credentials)
s.Ui = m.Ui
s.Mode = mode
return s
}
// process will process the meta-parameters out of the arguments. This

2
command/meta_new.go
View File

@ -45,7 +45,7 @@ func (m *Meta) Module(path string) (*module.Tree, error) {
return nil, err
}
err = mod.Load(m.moduleStorage(m.DataDir()), module.GetModeNone)
err = mod.Load(m.moduleStorage(m.DataDir(), module.GetModeNone))
if err != nil {
return nil, errwrap.Wrapf("Error loading modules: {{err}}", err)
}

29
command/module_storage.go
View File

@ -1,29 +0,0 @@
package command
import (
"fmt"
"github.com/hashicorp/go-getter"
"github.com/mitchellh/cli"
)
// uiModuleStorage implements module.Storage and is just a proxy to output
// to the UI any Get operations.
type uiModuleStorage struct {
Storage getter.Storage
Ui cli.Ui
}
func (s *uiModuleStorage) Dir(key string) (string, bool, error) {
return s.Storage.Dir(key)
}
func (s *uiModuleStorage) Get(key string, source string, update bool) error {
updateStr := ""
if update {
updateStr = " (update)"
}
s.Ui.Output(fmt.Sprintf("Get: %s%s", source, updateStr))
return s.Storage.Get(key, source, update)
}

11
command/module_storage_test.go
View File

@ -1,11 +0,0 @@
package command
import (
"testing"
"github.com/hashicorp/go-getter"
)
func TestUiModuleStorage_impl(t *testing.T) {
var _ getter.Storage = new(uiModuleStorage)
}

78
config/module/get_test.go
View File

@ -16,7 +16,6 @@ import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
"github.com/hashicorp/terraform/svchost/disco"
)
// Map of module names and location of test modules.
@ -26,6 +25,10 @@ type testMod struct {
version string
}
const (
testCredentials = "test-auth-token"
)
// All the locationes from the mockRegistry start with a file:// scheme. If
// the the location string here doesn't have a scheme, the mockRegistry will
// find the absolute path and return a complete URL.
@ -52,6 +55,9 @@ var testMods = map[string][]testMod{
{version: "1.2.2"},
{version: "1.2.1"},
},
"private/name/provider": {
{version: "1.0.0"},
},
}
func latestVersion(versions []string) string {
@ -82,6 +88,13 @@ func mockRegHandler() http.Handler {
return
}
// check for auth
if strings.Contains(matches[0], "private/") {
if !strings.Contains(r.Header.Get("Authorization"), testCredentials) {
http.Error(w, "", http.StatusForbidden)
}
}
versions, ok := testMods[matches[1]]
if !ok {
http.NotFound(w, r)
@ -111,6 +124,13 @@ func mockRegHandler() http.Handler {
return
}
// check for auth
if strings.Contains(matches[1], "private/") {
if !strings.Contains(r.Header.Get("Authorization"), testCredentials) {
http.Error(w, "", http.StatusForbidden)
}
}
name := matches[1]
versions, ok := testMods[name]
if !ok {
@ -162,7 +182,7 @@ func mockRegHandler() http.Handler {
mux.HandleFunc("/.well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, `{"modules.v1":"/v1/modules/"}`)
io.WriteString(w, `{"modules.v1":"http://localhost/v1/modules/"}`)
})
return mux
}
@ -174,29 +194,20 @@ func mockRegistry() *httptest.Server {
return server
}
func mockTLSRegistry() *httptest.Server {
server := httptest.NewTLSServer(mockRegHandler())
return server
}
// GitHub archives always contain the module source in a single subdirectory,
// so the registry will return a path with with a `//*` suffix. We need to make
// sure this doesn't intefere with our internal handling of `//` subdir.
func TestRegistryGitHubArchive(t *testing.T) {
server := mockTLSRegistry()
server := mockRegistry()
defer server.Close()
d := regDisco
regDisco = disco.NewDisco()
regDisco.Transport = mockTransport(server)
defer func() {
regDisco = d
}()
disco := testDisco(server)
storage := testStorage(t, disco)
storage := testStorage(t)
tree := NewTree("", testConfig(t, "registry-tar-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -204,7 +215,8 @@ func TestRegistryGitHubArchive(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -212,7 +224,8 @@ func TestRegistryGitHubArchive(t *testing.T) {
server.Close()
tree = NewTree("", testConfig(t, "registry-tar-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -229,20 +242,15 @@ func TestRegistryGitHubArchive(t *testing.T) {
// Test that the //subdir notation can be used with registry modules
func TestRegisryModuleSubdir(t *testing.T) {
server := mockTLSRegistry()
server := mockRegistry()
defer server.Close()
d := regDisco
regDisco = disco.NewDisco()
regDisco.Transport = mockTransport(server)
defer func() {
regDisco = d
}()
storage := testStorage(t)
disco := testDisco(server)
storage := testStorage(t, disco)
tree := NewTree("", testConfig(t, "registry-subdir"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -250,7 +258,8 @@ func TestRegisryModuleSubdir(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -272,7 +281,8 @@ func TestAccRegistryDiscover(t *testing.T) {
t.Fatal(err)
}
loc, err := lookupModuleLocation(nil, module, "")
s := NewStorage("/tmp", nil, nil)
loc, err := s.lookupModuleLocation(module, "")
if err != nil {
t.Fatal(err)
}
@ -296,10 +306,11 @@ func TestAccRegistryLoad(t *testing.T) {
t.Skip("skipping ACC test")
}
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "registry-load"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -307,7 +318,8 @@ func TestAccRegistryLoad(t *testing.T) {
t.Fatal("should be loaded")
}
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}

24
config/module/module_test.go
View File

@ -1,14 +1,17 @@
package module
import (
"fmt"
"io/ioutil"
"log"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/disco"
)
func init() {
@ -42,7 +45,22 @@ func testConfig(t *testing.T, n string) *config.Config {
return c
}
func testStorage(t *testing.T) getter.Storage {
func testStorage(t *testing.T, d *disco.Disco) *Storage {
t.Helper()
return &getter.FolderStorage{StorageDir: tempDir(t)}
return NewStorage(tempDir(t), d, nil)
}
// test discovery maps registry.terraform.io, localhost, localhost.localdomain,
// and example.com to the test server.
func testDisco(s *httptest.Server) *disco.Disco {
services := map[string]interface{}{
"modules.v1": fmt.Sprintf("%s/v1/modules/", s.URL),
}
d := disco.NewDisco()
d.ForceHostServices(svchost.Hostname("registry.terraform.io"), services)
d.ForceHostServices(svchost.Hostname("localhost"), services)
d.ForceHostServices(svchost.Hostname("localhost.localdomain"), services)
d.ForceHostServices(svchost.Hostname("example.com"), services)
return d
}

74
config/module/registry.go
View File

@ -16,7 +16,6 @@ import (
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/registry/response"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/hashicorp/terraform/version"
)
@ -33,7 +32,6 @@ const (
var (
httpClient *http.Client
tfVersion = version.String()
regDisco = disco.NewDisco()
)
func init() {
@ -47,16 +45,8 @@ func (e errModuleNotFound) Error() string {
return `module "` + string(e) + `" not found`
}
func discoverRegURL(d *disco.Disco, module *regsrc.Module) *url.URL {
if d == nil {
d = regDisco
}
if module.RawHost == nil {
module.RawHost = regsrc.NewFriendlyHost(defaultRegistry)
}
regURL := d.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID)
func (s *Storage) discoverRegURL(module *regsrc.Module) *url.URL {
regURL := s.Services.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID)
if regURL == nil {
regURL = &url.URL{
Scheme: "https",
@ -72,9 +62,29 @@ func discoverRegURL(d *disco.Disco, module *regsrc.Module) *url.URL {
return regURL
}
func (s *Storage) addRequestCreds(host svchost.Hostname, req *http.Request) {
if s.Creds == nil {
return
}
creds, err := s.Creds.ForHost(host)
if err != nil {
log.Printf("[WARNING] Failed to get credentials for %s: %s (ignoring)", host, err)
return
}
if creds != nil {
creds.PrepareRequest(req)
}
}
// Lookup module versions in the registry.
func lookupModuleVersions(d *disco.Disco, module *regsrc.Module) (*response.ModuleVersions, error) {
service := discoverRegURL(d, module)
func (s *Storage) lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) {
if module.RawHost == nil {
module.RawHost = regsrc.NewFriendlyHost(defaultRegistry)
}
service := s.discoverRegURL(module)
p, err := url.Parse(path.Join(module.Module(), "versions"))
if err != nil {
@ -90,22 +100,10 @@ func lookupModuleVersions(d *disco.Disco, module *regsrc.Module) (*response.Modu
return nil, err
}
s.addRequestCreds(svchost.Hostname(module.RawHost.Normalized()), req)
req.Header.Set(xTerraformVersion, tfVersion)
if d == nil {
d = regDisco
}
// if discovery required a custom transport, then we should use that too
client := httpClient
if d.Transport != nil {
client = &http.Client{
Transport: d.Transport,
Timeout: requestTimeout,
}
}
resp, err := client.Do(req)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
@ -131,8 +129,12 @@ func lookupModuleVersions(d *disco.Disco, module *regsrc.Module) (*response.Modu
}
// lookup the location of a specific module version in the registry
func lookupModuleLocation(d *disco.Disco, module *regsrc.Module, version string) (string, error) {
service := discoverRegURL(d, module)
func (s *Storage) lookupModuleLocation(module *regsrc.Module, version string) (string, error) {
if module.RawHost == nil {
module.RawHost = regsrc.NewFriendlyHost(defaultRegistry)
}
service := s.discoverRegURL(module)
var p *url.URL
var err error
@ -153,18 +155,10 @@ func lookupModuleLocation(d *disco.Disco, module *regsrc.Module, version string)
return "", err
}
s.addRequestCreds(svchost.Hostname(module.RawHost.Normalized()), req)
req.Header.Set(xTerraformVersion, tfVersion)
// if discovery required a custom transport, then we should use that too
client := httpClient
if regDisco.Transport != nil {
client = &http.Client{
Transport: regDisco.Transport,
Timeout: requestTimeout,
}
}
resp, err := client.Do(req)
resp, err := httpClient.Do(req)
if err != nil {
return "", err
}

110
config/module/registry_test.go
View File

@ -1,75 +1,21 @@
package module
import (
"context"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
cleanhttp "github.com/hashicorp/go-cleanhttp"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/svchost"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
)
// Return a transport to use for this test server.
// This not only loads the tls.Config from the test server for proper cert
// validation, but also inserts a Dialer that resolves localhost and
// example.com to 127.0.0.1 with the correct port, since 127.0.0.1 on its own
// isn't a valid registry hostname.
// TODO: cert validation not working here, so we use don't verify for now.
func mockTransport(server *httptest.Server) *http.Transport {
u, _ := url.Parse(server.URL)
_, port, _ := net.SplitHostPort(u.Host)
transport := cleanhttp.DefaultTransport()
transport.TLSClientConfig = server.TLS
transport.TLSClientConfig.InsecureSkipVerify = true
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
host, _, _ := net.SplitHostPort(addr)
switch host {
case "example.com", "localhost", "localhost.localdomain", "registry.terraform.io":
addr = "127.0.0.1"
if port != "" {
addr += ":" + port
}
}
return (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext(ctx, network, addr)
}
return transport
}
func TestMockDiscovery(t *testing.T) {
server := mockTLSRegistry()
defer server.Close()
regDisco := disco.NewDisco()
regDisco.Transport = mockTransport(server)
regURL := regDisco.DiscoverServiceURL("example.com", serviceID)
if regURL == nil {
t.Fatal("no registry service discovered")
}
if regURL.Host != "example.com" {
t.Fatal("expected registry host example.com, got:", regURL.Host)
}
}
func TestLookupModuleVersions(t *testing.T) {
server := mockTLSRegistry()
server := mockRegistry()
defer server.Close()
regDisco := disco.NewDisco()
regDisco.Transport = mockTransport(server)
regDisco := testDisco(server)
// test with and without a hostname
for _, src := range []string{
@ -81,7 +27,8 @@ func TestLookupModuleVersions(t *testing.T) {
t.Fatal(err)
}
resp, err := lookupModuleVersions(regDisco, modsrc)
s := &Storage{Services: regDisco}
resp, err := s.lookupModuleVersions(modsrc)
if err != nil {
t.Fatal(err)
}
@ -109,6 +56,44 @@ func TestLookupModuleVersions(t *testing.T) {
}
}
func TestRegistryAuth(t *testing.T) {
server := mockRegistry()
defer server.Close()
regDisco := testDisco(server)
storage := testStorage(t, regDisco)
src := "private/name/provider"
mod, err := regsrc.ParseModuleSource(src)
if err != nil {
t.Fatal(err)
}
// both should fail without auth
_, err = storage.lookupModuleVersions(mod)
if err == nil {
t.Fatal("expected error")
}
_, err = storage.lookupModuleLocation(mod, "1.0.0")
if err == nil {
t.Fatal("expected error")
}
storage.Creds = auth.StaticCredentialsSource(map[svchost.Hostname]map[string]interface{}{
svchost.Hostname(defaultRegistry): {"token": testCredentials},
})
_, err = storage.lookupModuleVersions(mod)
if err != nil {
t.Fatal(err)
}
_, err = storage.lookupModuleLocation(mod, "1.0.0")
if err != nil {
t.Fatal(err)
}
}
func TestAccLookupModuleVersions(t *testing.T) {
if os.Getenv("TF_ACC") == "" {
t.Skip()
@ -125,7 +110,10 @@ func TestAccLookupModuleVersions(t *testing.T) {
t.Fatal(err)
}
resp, err := lookupModuleVersions(regDisco, modsrc)
s := &Storage{
Services: regDisco,
}
resp, err := s.lookupModuleVersions(modsrc)
if err != nil {
t.Fatal(err)
}

125
config/module/storage.go
View File

@ -7,10 +7,12 @@ import (
"log"
"os"
"path/filepath"
"reflect"
getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/registry/regsrc"
"github.com/hashicorp/terraform/svchost/auth"
"github.com/hashicorp/terraform/svchost/disco"
"github.com/mitchellh/cli"
)
const manifestName = "modules.json"
@ -54,49 +56,44 @@ type moduleRecord struct {
registry bool
}
// moduleStorage implements methods to record and fetch metadata about the
// modules that have been fetched and stored locally. The getter.Storgae
// abstraction doesn't provide the information needed to know which versions of
// a module have been stored, or their location.
type moduleStorage struct {
getter.Storage
storageDir string
mode GetMode
// Storage implements methods to manage the storage of modules.
// This is used by Tree.Load to query registries, authenticate requests, and
// store modules locally.
type Storage struct {
// StorageDir is the full path to the directory where all modules will be
// stored.
StorageDir string
// Services is a required *disco.Disco, which may have services and
// credentials pre-loaded.
Services *disco.Disco
// Creds optionally provides credentials for communicating with service
// providers.
Creds auth.CredentialsSource
// Ui is an optional cli.Ui for user output
Ui cli.Ui
// Mode is the GetMode that will be used for various operations.
Mode GetMode
}
func newModuleStorage(s getter.Storage, mode GetMode) moduleStorage {
return moduleStorage{
Storage: s,
storageDir: storageDir(s),
mode: mode,
}
func NewStorage(dir string, services *disco.Disco, creds auth.CredentialsSource) *Storage {
s := &Storage{
StorageDir: dir,
Services: services,
Creds: creds,
}
// The Tree needs to know where to store the module manifest.
// Th Storage abstraction doesn't provide access to the storage root directory,
// so we extract it here.
func storageDir(s getter.Storage) string {
// get the StorageDir directly if possible
switch t := s.(type) {
case *getter.FolderStorage:
return t.StorageDir
case moduleStorage:
return t.storageDir
case nil:
return ""
// make sure this isn't nil
if s.Services == nil {
s.Services = disco.NewDisco()
}
// this should be our UI wrapper which is exported here, so we need to
// extract the FolderStorage via reflection.
fs := reflect.ValueOf(s).Elem().FieldByName("Storage").Interface()
return storageDir(fs.(getter.Storage))
return s
}
// loadManifest returns the moduleManifest file from the parent directory.
func (m moduleStorage) loadManifest() (moduleManifest, error) {
func (s Storage) loadManifest() (moduleManifest, error) {
manifest := moduleManifest{}
manifestPath := filepath.Join(m.storageDir, manifestName)
manifestPath := filepath.Join(s.StorageDir, manifestName)
data, err := ioutil.ReadFile(manifestPath)
if err != nil && !os.IsNotExist(err) {
return manifest, err
@ -116,8 +113,8 @@ func (m moduleStorage) loadManifest() (moduleManifest, error) {
// root directory. The storage method loads the entire file and rewrites it
// each time. This is only done a few times during init, so efficiency is
// not a concern.
func (m moduleStorage) recordModule(rec moduleRecord) error {
manifest, err := m.loadManifest()
func (s Storage) recordModule(rec moduleRecord) error {
manifest, err := s.loadManifest()
if err != nil {
// if there was a problem with the file, we will attempt to write a new
// one. Any non-data related error should surface there.
@ -146,15 +143,15 @@ func (m moduleStorage) recordModule(rec moduleRecord) error {
panic(err)
}
manifestPath := filepath.Join(m.storageDir, manifestName)
manifestPath := filepath.Join(s.StorageDir, manifestName)
return ioutil.WriteFile(manifestPath, js, 0644)
}
// load the manifest from dir, and return all module versions matching the
// provided source. Records with no version info will be skipped, as they need
// to be uniquely identified by other means.
func (m moduleStorage) moduleVersions(source string) ([]moduleRecord, error) {
manifest, err := m.loadManifest()
func (s Storage) moduleVersions(source string) ([]moduleRecord, error) {
manifest, err := s.loadManifest()
if err != nil {
return manifest.Modules, err
}
@ -170,8 +167,8 @@ func (m moduleStorage) moduleVersions(source string) ([]moduleRecord, error) {
return matching, nil
}
func (m moduleStorage) moduleDir(key string) (string, error) {
manifest, err := m.loadManifest()
func (s Storage) moduleDir(key string) (string, error) {
manifest, err := s.loadManifest()
if err != nil {
return "", err
}
@ -186,8 +183,8 @@ func (m moduleStorage) moduleDir(key string) (string, error) {
}
// return only the root directory of the module stored in dir.
func (m moduleStorage) getModuleRoot(dir string) (string, error) {
manifest, err := m.loadManifest()
func (s Storage) getModuleRoot(dir string) (string, error) {
manifest, err := s.loadManifest()
if err != nil {
return "", err
}
@ -201,42 +198,53 @@ func (m moduleStorage) getModuleRoot(dir string) (string, error) {
}
// record only the Root directory for the module stored at dir.
// TODO: remove this compatibility function to store the full moduleRecord.
func (m moduleStorage) recordModuleRoot(dir, root string) error {
func (s Storage) recordModuleRoot(dir, root string) error {
rec := moduleRecord{
Dir: dir,
Root: root,
}
return m.recordModule(rec)
return s.recordModule(rec)
}
func (s Storage) getStorage(key string, src string) (string, bool, error) {
storage := &getter.FolderStorage{
StorageDir: s.StorageDir,
}
if s.Ui != nil {
update := ""
if s.Mode == GetModeUpdate {
update = " (update)"
}
s.Ui.Output(fmt.Sprintf("Get: %s%s", src, update))
}
func (m moduleStorage) getStorage(key string, src string) (string, bool, error) {
// Get the module with the level specified if we were told to.
if m.mode > GetModeNone {
if s.Mode > GetModeNone {
log.Printf("[DEBUG] fetching %q with key %q", src, key)
if err := m.Storage.Get(key, src, m.mode == GetModeUpdate); err != nil {
if err := storage.Get(key, src, s.Mode == GetModeUpdate); err != nil {
return "", false, err
}
}
// Get the directory where the module is.
dir, found, err := m.Storage.Dir(key)
dir, found, err := storage.Dir(key)
log.Printf("[DEBUG] found %q in %q: %t", src, dir, found)
return dir, found, err
}
// find a stored module that's not from a registry
func (m moduleStorage) findModule(key string) (string, error) {
if m.mode == GetModeUpdate {
func (s Storage) findModule(key string) (string, error) {
if s.Mode == GetModeUpdate {
return "", nil
}
return m.moduleDir(key)
return s.moduleDir(key)
}
// find a registry module
func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRecord, error) {
func (s Storage) findRegistryModule(mSource, constraint string) (moduleRecord, error) {
rec := moduleRecord{
Source: mSource,
}
@ -254,7 +262,7 @@ func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRec
log.Printf("[TRACE] %q is a registry module", mod.Module())
versions, err := m.moduleVersions(mod.String())
versions, err := s.moduleVersions(mod.String())
if err != nil {
log.Printf("[ERROR] error looking up versions for %q: %s", mod.Module(), err)
return rec, err
@ -262,7 +270,6 @@ func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRec
match, err := newestRecord(versions, constraint)
if err != nil {
// TODO: does this allow previously unversioned modules?
log.Printf("[INFO] no matching version for %q<%s>, %s", mod.Module(), constraint, err)
}
@ -272,8 +279,8 @@ func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRec
// we need to lookup available versions
// Only on Get if it's not found, on unconditionally on Update
if (m.mode == GetModeGet && !found) || (m.mode == GetModeUpdate) {
resp, err := lookupModuleVersions(nil, mod)
if (s.Mode == GetModeGet && !found) || (s.Mode == GetModeUpdate) {
resp, err := s.lookupModuleVersions(mod)
if err != nil {
return rec, err
}
@ -293,7 +300,7 @@ func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRec
rec.Version = match.Version
rec.url, err = lookupModuleLocation(nil, mod, rec.Version)
rec.url, err = s.lookupModuleLocation(mod, rec.Version)
if err != nil {
return rec, err
}

6
config/module/testing.go
View File

@ -4,8 +4,6 @@ import (
"io/ioutil"
"os"
"testing"
"github.com/hashicorp/go-getter"
)
// TestTree loads a module at the given path and returns the tree as well
@ -26,8 +24,8 @@ func TestTree(t *testing.T, path string) (*Tree, func()) {
}
// Get the child modules
s := &getter.FolderStorage{StorageDir: dir}
if err := mod.Load(s, GetModeGet); err != nil {
s := &Storage{StorageDir: dir, Mode: GetModeGet}
if err := mod.Load(s); err != nil {
t.Fatalf("err: %s", err)
return nil, nil
}

10
config/module/tree.go
View File

@ -52,7 +52,7 @@ func NewEmptyTree() *Tree {
// We do this dummy load so that the tree is marked as "loaded". It
// should never fail because this is just about a no-op. If it does fail
// we panic so we can know its a bug.
if err := t.Load(nil, GetModeGet); err != nil {
if err := t.Load(&Storage{Mode: GetModeGet}); err != nil {
panic(err)
}
@ -169,12 +169,10 @@ func (t *Tree) Name() string {
// module trees inherently require the configuration to be in a reasonably
// sane state: no circular dependencies, proper module sources, etc. A full
// suite of validations can be done by running Validate (after loading).
func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
func (t *Tree) Load(s *Storage) error {
t.lock.Lock()
defer t.lock.Unlock()
s := newModuleStorage(storage, mode)
children, err := t.getChildren(s)
if err != nil {
return err
@ -182,7 +180,7 @@ func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
// Go through all the children and load them.
for _, c := range children {
if err := c.Load(storage, mode); err != nil {
if err := c.Load(s); err != nil {
return err
}
}
@ -198,7 +196,7 @@ func (t *Tree) Load(storage getter.Storage, mode GetMode) error {
return nil
}
func (t *Tree) getChildren(s moduleStorage) (map[string]*Tree, error) {
func (t *Tree) getChildren(s *Storage) (map[string]*Tree, error) {
children := make(map[string]*Tree)
// Go through all the modules and get the directory for them.

5
config/module/tree_gob_test.go
View File

@ -8,11 +8,12 @@ import (
)
func TestTreeEncodeDecodeGob(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "basic"))
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}

133
config/module/tree_test.go
View File

@ -10,7 +10,6 @@ import (
"strings"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/helper/copy"
)
@ -21,9 +20,10 @@ func TestTreeChild(t *testing.T) {
t.Fatal("child should be nil")
}
storage := testStorage(t)
storage := testStorage(t, nil)
storage.Mode = GetModeGet
tree := NewTree("", testConfig(t, "child"))
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -65,7 +65,7 @@ func TestTreeChild(t *testing.T) {
}
func TestTreeLoad(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "basic"))
if tree.Loaded() {
@ -73,7 +73,7 @@ func TestTreeLoad(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -82,7 +82,8 @@ func TestTreeLoad(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -91,7 +92,8 @@ func TestTreeLoad(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -103,7 +105,7 @@ func TestTreeLoad(t *testing.T) {
}
func TestTreeLoad_duplicate(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "dup"))
if tree.Loaded() {
@ -111,19 +113,23 @@ func TestTreeLoad_duplicate(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err == nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err == nil {
t.Fatalf("should error")
}
}
func TestTreeLoad_copyable(t *testing.T) {
dir := tempDir(t)
storage := &getter.FolderStorage{StorageDir: dir}
storage := &Storage{
StorageDir: dir,
Mode: GetModeGet,
}
cfg := testConfig(t, "basic")
tree := NewTree("", cfg)
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -132,7 +138,8 @@ func TestTreeLoad_copyable(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -160,10 +167,13 @@ func TestTreeLoad_copyable(t *testing.T) {
}
tree := NewTree("", cfg)
storage := &getter.FolderStorage{StorageDir: dir2}
storage := &Storage{
StorageDir: dir2,
Mode: GetModeNone,
}
// This should not error since we already got it!
if err := tree.Load(storage, GetModeNone); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -174,7 +184,7 @@ func TestTreeLoad_copyable(t *testing.T) {
}
func TestTreeLoad_parentRef(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "basic-parent"))
if tree.Loaded() {
@ -182,7 +192,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -191,7 +202,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -200,7 +212,8 @@ func TestTreeLoad_parentRef(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -215,17 +228,12 @@ func TestTreeLoad_subdir(t *testing.T) {
fixtures := []string{
"basic-subdir",
"basic-tar-subdir",
// Passing a subpath to go getter extracts only this subpath. The old
// internal code would keep the entire directory structure, allowing a
// top-level module to reference others through its parent directory.
// TODO: this can be removed as a breaking change in a major release.
"tar-subdir-to-parent",
}
for _, tc := range fixtures {
t.Run(tc, func(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, tc))
if tree.Loaded() {
@ -233,7 +241,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should error because we haven't gotten things yet
if err := tree.Load(storage, GetModeNone); err == nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err == nil {
t.Fatal("should error")
}
@ -242,7 +251,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should get things
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -251,7 +261,8 @@ func TestTreeLoad_subdir(t *testing.T) {
}
// This should no longer error
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -271,7 +282,7 @@ func TestTree_recordManifest(t *testing.T) {
}
defer os.RemoveAll(td)
storage := moduleStorage{storageDir: td}
storage := Storage{StorageDir: td}
dir := filepath.Join(td, "0131bf0fef686e090b16bdbab4910ddf")
@ -388,7 +399,9 @@ func TestTreeValidate_table(t *testing.T) {
for i, tc := range cases {
t.Run(fmt.Sprintf("%d-%s", i, tc.Name), func(t *testing.T) {
tree := NewTree("", testConfig(t, tc.Fixture))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -409,7 +422,9 @@ func TestTreeValidate_table(t *testing.T) {
func TestTreeValidate_badChild(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-child-bad"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -421,7 +436,9 @@ func TestTreeValidate_badChild(t *testing.T) {
func TestTreeValidate_badChildOutput(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-output"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -433,7 +450,9 @@ func TestTreeValidate_badChildOutput(t *testing.T) {
func TestTreeValidate_badChildOutputToModule(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-output-to-module"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -445,7 +464,9 @@ func TestTreeValidate_badChildOutputToModule(t *testing.T) {
func TestTreeValidate_badChildVar(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-bad-var"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -457,7 +478,9 @@ func TestTreeValidate_badChildVar(t *testing.T) {
func TestTreeValidate_badRoot(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-root-bad"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -469,7 +492,9 @@ func TestTreeValidate_badRoot(t *testing.T) {
func TestTreeValidate_good(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-child-good"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -489,7 +514,9 @@ func TestTreeValidate_notLoaded(t *testing.T) {
func TestTreeValidate_requiredChildVar(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-required-var"))
if err := tree.Load(testStorage(t), GetModeGet); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -510,7 +537,9 @@ func TestTreeValidate_requiredChildVar(t *testing.T) {
func TestTreeValidate_unknownModule(t *testing.T) {
tree := NewTree("", testConfig(t, "validate-module-unknown"))
if err := tree.Load(testStorage(t), GetModeNone); err != nil {
storage := testStorage(t, nil)
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -520,10 +549,11 @@ func TestTreeValidate_unknownModule(t *testing.T) {
}
func TestTreeProviders_basic(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "basic-parent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -582,10 +612,11 @@ func TestTreeProviders_basic(t *testing.T) {
}
func TestTreeProviders_implicit(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "implicit-parent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -622,10 +653,11 @@ func TestTreeProviders_implicit(t *testing.T) {
}
func TestTreeProviders_implicitMultiLevel(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "implicit-grandparent-providers"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("err: %s", err)
}
@ -691,10 +723,11 @@ func TestTreeProviders_implicitMultiLevel(t *testing.T) {
}
func TestTreeLoad_conflictingSubmoduleNames(t *testing.T) {
storage := testStorage(t)
storage := testStorage(t, nil)
tree := NewTree("", testConfig(t, "conficting-submodule-names"))
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}
@ -703,7 +736,8 @@ func TestTreeLoad_conflictingSubmoduleNames(t *testing.T) {
}
// Try to reload
if err := tree.Load(storage, GetModeNone); err != nil {
storage.Mode = GetModeNone
if err := tree.Load(storage); err != nil {
t.Fatalf("reload failed: %s", err)
}
@ -753,13 +787,14 @@ func TestTreeLoad_changeIntermediateSource(t *testing.T) {
if err := os.MkdirAll(".terraform/modules", 0777); err != nil {
t.Fatal(err)
}
storage := &getter.FolderStorage{StorageDir: ".terraform/modules"}
storage := &Storage{StorageDir: ".terraform/modules"}
cfg, err := config.LoadDir("./")
if err != nil {
t.Fatal(err)
}
tree := NewTree("", cfg)
if err := tree.Load(storage, GetModeGet); err != nil {
storage.Mode = GetModeGet
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}
@ -774,7 +809,7 @@ func TestTreeLoad_changeIntermediateSource(t *testing.T) {
t.Fatal(err)
}
tree = NewTree("", cfg)
if err := tree.Load(storage, GetModeGet); err != nil {
if err := tree.Load(storage); err != nil {
t.Fatalf("load failed: %s", err)
}

6
helper/resource/testing.go
View File

@ -15,7 +15,6 @@ import (
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/logutils"
"github.com/hashicorp/terraform/config/module"
@ -751,10 +750,11 @@ func testModule(
}
// Load the modules
modStorage := &getter.FolderStorage{
modStorage := &module.Storage{
StorageDir: filepath.Join(cfgPath, ".tfmodules"),
Mode: module.GetModeGet,
}
err = mod.Load(modStorage, module.GetModeGet)
err = mod.Load(modStorage)
if err != nil {
return nil, fmt.Errorf("Error downloading modules: %s", err)
}

13
terraform/terraform_test.go
View File

@ -12,7 +12,6 @@ import (
"sync"
"testing"
"github.com/hashicorp/go-getter"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/helper/experiment"
@ -97,8 +96,11 @@ func testModule(t *testing.T, name string) *module.Tree {
t.Fatalf("err: %s", err)
}
s := &getter.FolderStorage{StorageDir: tempDir(t)}
if err := mod.Load(s, module.GetModeGet); err != nil {
s := &module.Storage{
StorageDir: tempDir(t),
Mode: module.GetModeGet,
}
if err := mod.Load(s); err != nil {
t.Fatalf("err: %s", err)
}
@ -144,10 +146,11 @@ func testModuleInline(t *testing.T, config map[string]string) *module.Tree {
}
// Load the modules
modStorage := &getter.FolderStorage{
modStorage := &module.Storage{
StorageDir: filepath.Join(cfgPath, ".tfmodules"),
Mode: module.GetModeGet,
}
err = mod.Load(modStorage, module.GetModeGet)
err = mod.Load(modStorage)
if err != nil {
t.Errorf("Error downloading modules: %s", err)
}