config/module: HTTP supports subdirs

config/module/copy_dir.go

@@ -0,0 +1,51 @@
+package module
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+// copyDir copies the src directory contents into dst. Both directories
+// should already exist.
+func copyDir(dst, src string) error {
+	walkFn := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		dstPath := filepath.Join(dst, filepath.Base(path))
+
+		// If we have a directory, make that subdirectory, then continue
+		// the walk.
+		if info.IsDir() {
+			if err := os.MkdirAll(dstPath, 0755); err != nil {
+				return err
+			}
+
+			return copyDir(dstPath, path)
+		}
+
+		// If we have a file, copy the contents.
+		srcF, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer srcF.Close()
+
+		dstF, err := os.Create(dstPath)
+		if err != nil {
+			return err
+		}
+		defer dstF.Close()
+
+		if _, err := io.Copy(dstF, srcF); err != nil {
+			return err
+		}
+
+		// Chmod it
+		return os.Chmod(dstPath, info.Mode())
+	}
+
+	return filepath.Walk(src, walkFn)
+}

config/module/get.go

@@ -100,12 +100,20 @@ func getRunCommand(cmd *exec.Cmd) error {
 // getDirSubdir takes a source and returns a tuple of the URL without
 // the subdir and the URL with the subdir.
 func getDirSubdir(src string) (string, string) {
+	// Calcaulate an offset to avoid accidentally marking the scheme
+	// as the dir.
+	var offset int
+	if idx := strings.Index(src, "://"); idx > -1 {
+		offset = idx + 3
+	}
+
 	// First see if we even have an explicit subdir
-	idx := strings.Index(src, "//")
+	idx := strings.Index(src[offset:], "//")
 	if idx == -1 {
 		return src, ""
 	}
 
+	idx += offset
 	subdir := src[idx+2:]
 	src = src[:idx]
 

config/module/get_http.go

@@ -4,8 +4,11 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
 	"strings"
 )
 
@@ -62,8 +65,50 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error {
 		return fmt.Errorf("no source URL was returned")
 	}
 
-	// Get it!
-	return Get(dst, source)
+	// If there is a subdir component, then we download the root separately
+	// into a temporary directory, then copy over the proper subdir.
+	source, subDir := getDirSubdir(source)
+	if subDir == "" {
+		return Get(dst, source)
+	}
+
+	// We have a subdir, time to jump some hoops
+	return g.getSubdir(dst, source, subDir)
+}
+
+// getSubdir downloads the source into the destination, but with
+// the proper subdir.
+func (g *HttpGetter) getSubdir(dst, source, subDir string) error {
+	// Create a temporary directory to store the full source
+	td, err := ioutil.TempDir("", "tf")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(td)
+
+	// Download that into the given directory
+	if err := Get(td, source); err != nil {
+		return err
+	}
+
+	// Make sure the subdir path actually exists
+	sourcePath := filepath.Join(td, subDir)
+	if _, err := os.Stat(sourcePath); err != nil {
+		return fmt.Errorf(
+			"Error downloading %s: %s", source, err)
+	}
+
+	// Copy the subdirectory into our actual destination.
+	if err := os.RemoveAll(dst); err != nil {
+		return err
+	}
+
+	// Make the final destination
+	if err := os.MkdirAll(dst, 0755); err != nil {
+		return err
+	}
+
+	return copyDir(dst, sourcePath)
 }
 
 // parseMeta looks for the first meta tag in the given reader that

config/module/get_http_test.go

@@ -62,6 +62,30 @@ func TestHttpGetter_meta(t *testing.T) {
 	}
 }
 
+func TestHttpGetter_metaSubdir(t *testing.T) {
+	ln := testHttpServer(t)
+	defer ln.Close()
+
+	g := new(HttpGetter)
+	dst := tempDir(t)
+
+	var u url.URL
+	u.Scheme = "http"
+	u.Host = ln.Addr().String()
+	u.Path = "/meta-subdir"
+
+	// Get it!
+	if err := g.Get(dst, &u); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Verify the main file exists
+	mainPath := filepath.Join(dst, "sub.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func TestHttpGetter_none(t *testing.T) {
 	ln := testHttpServer(t)
 	defer ln.Close()
@@ -89,6 +113,7 @@ func testHttpServer(t *testing.T) net.Listener {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/header", testHttpHandlerHeader)
 	mux.HandleFunc("/meta", testHttpHandlerMeta)
+	mux.HandleFunc("/meta-subdir", testHttpHandlerMetaSubdir)
 
 	var server http.Server
 	server.Handler = mux
@@ -106,6 +131,10 @@ func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String())))
 }
 
+func testHttpHandlerMetaSubdir(w http.ResponseWriter, r *http.Request) {
+	w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//subdir").String())))
+}
+
 func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(testHttpNoneStr))
 }

config/module/get_test.go

@@ -64,8 +64,8 @@ func TestGetDirSubdir(t *testing.T) {
 			"hashicorp.com?bar=baz", "foo",
 		},
 		{
-			"hashicorp.com",
-			"hashicorp.com", "",
+			"file://foo//bar",
+			"file://foo", "bar",
 		},
 	}