config: support subdirectories with the "//" syntax

config/module/get.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"os/exec"
 	"regexp"
+	"strings"
 	"syscall"
 )
 
@@ -96,6 +97,29 @@ func getRunCommand(cmd *exec.Cmd) error {
 	return fmt.Errorf("error running %s: %s", cmd.Path, buf.String())
 }
 
+// getDirSubdir takes a source and returns a tuple of the URL without
+// the subdir and the URL with the subdir.
+func getDirSubdir(src string) (string, string) {
+	// First see if we even have an explicit subdir
+	idx := strings.Index(src, "//")
+	if idx == -1 {
+		return src, ""
+	}
+
+	subdir := src[idx+2:]
+	src = src[:idx]
+
+	// Next, check if we have query parameters and push them onto the
+	// URL.
+	if idx = strings.Index(subdir, "?"); idx > -1 {
+		query := subdir[idx:]
+		subdir = subdir[:idx]
+		src += query
+	}
+
+	return src, subdir
+}
+
 // getForcedGetter takes a source and returns the tuple of the forced
 // getter and the raw URL (without the force syntax).
 func getForcedGetter(src string) (string, string) {

config/module/get_test.go

@@ -45,3 +45,37 @@ func TestGet_fileForced(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 }
+
+func TestGetDirSubdir(t *testing.T) {
+	cases := []struct {
+		Input string
+		Dir, Sub string
+	}{
+		{
+			"hashicorp.com",
+			"hashicorp.com", "",
+		},
+		{
+			"hashicorp.com//foo",
+			"hashicorp.com", "foo",
+		},
+		{
+			"hashicorp.com//foo?bar=baz",
+			"hashicorp.com?bar=baz", "foo",
+		},
+		{
+			"hashicorp.com",
+			"hashicorp.com", "",
+		},
+	}
+
+	for i, tc := range cases {
+		adir, asub := getDirSubdir(tc.Input)
+		if adir != tc.Dir {
+			t.Fatalf("%d: bad dir: %#v", i, adir)
+		}
+		if asub != tc.Sub {
+			t.Fatalf("%d: bad sub: %#v", i, asub)
+		}
+	}
+}

config/module/storage.go

@@ -11,3 +11,15 @@ type Storage interface {
 	// Get will download and optionally update the given module.
 	Get(string, bool) error
 }
+
+func getStorage(s Storage, src string, mode GetMode) (string, bool, error) {
+	// Get the module with the level specified if we were told to.
+	if mode > GetModeNone {
+		if err := s.Get(src, mode == GetModeUpdate); err != nil {
+			return "", false, err
+		}
+	}
+
+	// Get the directory where the module is.
+	return s.Dir(src)
+}

config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG

@@ -1,7 +1,7 @@
-Branch
+add subdir
 # Please enter the commit message for your changes. Lines starting
 # with '#' will be ignored, and an empty message aborts the commit.
-# On branch test-branch
+# On branch master
 # Changes to be committed:
-#	new file:   main_branch.tf
+#	new file:   subdir/sub.tf
 #

config/module/test-fixtures/basic-git/DOTgit/logs/HEAD

@@ -4,3 +4,4 @@
 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410886909 -0700	checkout: moving from master to test-branch
 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410886913 -0700	commit: Branch
 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410886916 -0700	checkout: moving from test-branch to master
+1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1411767116 -0700	commit: add subdir

config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master

@@ -1,3 +1,4 @@
 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410850637 -0700	commit (initial): A commit
 497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410886526 -0700	commit: tag1
 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1410886536 -0700	commit: remove tag1
+1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto <mitchell.hashimoto@gmail.com> 1411767116 -0700	commit: add subdir

config/module/test-fixtures/basic-git/DOTgit/refs/heads/master

@@ -1 +1 @@
-1f31e97f053caeb5d6b7bffa3faf82941c99efa2
+146492b04efe0aae2b8288c5c0aef6a951030fde

config/module/test-fixtures/basic-subdir/foo/sub/main.tf

@@ -0,0 +1,3 @@
+module "bar" {
+    source = "./baz"
+}

config/module/test-fixtures/basic-subdir/main.tf

@@ -0,0 +1,3 @@
+module "foo" {
+    source = "./foo//sub"
+}

config/module/tree.go

@@ -2,6 +2,7 @@ package module
 
 import (
 	"bufio"
+	"path/filepath"
 	"bytes"
 	"fmt"
 	"strings"
@@ -131,27 +132,22 @@ func (t *Tree) Load(s Storage, mode GetMode) error {
 	children := make(map[string]*Tree)
 
 	// Go through all the modules and get the directory for them.
-	update := mode == GetModeUpdate
 	for _, m := range modules {
 		if _, ok := children[m.Name]; ok {
 			return fmt.Errorf(
 				"module %s: duplicated. module names must be unique", m.Name)
 		}
 
-		source, err := Detect(m.Source, t.config.Dir)
+		// Split out the subdir if we have one
+		source, subDir := getDirSubdir(m.Source)
+
+		source, err := Detect(source, t.config.Dir)
 		if err != nil {
 			return fmt.Errorf("module %s: %s", m.Name, err)
 		}
 
-		if mode > GetModeNone {
-			// Get the module since we specified we should
-			if err := s.Get(source, update); err != nil {
-				return err
-			}
-		}
-
 		// Get the directory where this module is so we can load it
-		dir, ok, err := s.Dir(source)
+		dir, ok, err := getStorage(s, source, mode)
 		if err != nil {
 			return err
 		}
@@ -160,7 +156,12 @@ func (t *Tree) Load(s Storage, mode GetMode) error {
 				"module %s: not found, may need to be downloaded", m.Name)
 		}
 
-		// Load the configuration
+		// If we have a subdirectory, then merge that in
+		if subDir != "" {
+			dir = filepath.Join(dir, subDir)
+		}
+
+		// Load the configurations.Dir(source)
 		children[m.Name], err = NewTreeModule(m.Name, dir)
 		if err != nil {
 			return fmt.Errorf(

config/module/tree_test.go

@@ -58,6 +58,44 @@ func TestTreeLoad_duplicate(t *testing.T) {
 	}
 }
 
+func TestTreeLoad_subdir(t *testing.T) {
+	storage := testStorage(t)
+	tree := NewTree("", testConfig(t, "basic-subdir"))
+
+	if tree.Loaded() {
+		t.Fatal("should not be loaded")
+	}
+
+	// This should error because we haven't gotten things yet
+	if err := tree.Load(storage, GetModeNone); err == nil {
+		t.Fatal("should error")
+	}
+
+	if tree.Loaded() {
+		t.Fatal("should not be loaded")
+	}
+
+	// This should get things
+	if err := tree.Load(storage, GetModeGet); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !tree.Loaded() {
+		t.Fatal("should be loaded")
+	}
+
+	// This should no longer error
+	if err := tree.Load(storage, GetModeNone); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	actual := strings.TrimSpace(tree.String())
+	expected := strings.TrimSpace(treeLoadSubdirStr)
+	if actual != expected {
+		t.Fatalf("bad: \n\n%s", actual)
+	}
+}
+
 func TestTreeModules(t *testing.T) {
 	tree := NewTree("", testConfig(t, "basic"))
 	actual := tree.Modules()
@@ -164,3 +202,9 @@ const treeLoadStr = `
 root
   foo
 `
+
+const treeLoadSubdirStr = `
+root
+  foo
+    bar
+`