From 7aa30fdf25d6456343a725d13ae40c3775f61735 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 26 Sep 2014 14:56:04 -0700 Subject: [PATCH 1/4] config: support subdirectories with the "//" syntax --- config/module/get.go | 24 ++++++++++ config/module/get_test.go | 34 ++++++++++++++ config/module/storage.go | 12 +++++ .../basic-git/DOTgit/COMMIT_EDITMSG | 6 +-- .../test-fixtures/basic-git/DOTgit/index | Bin 104 -> 184 bytes .../test-fixtures/basic-git/DOTgit/logs/HEAD | 1 + .../basic-git/DOTgit/logs/refs/heads/master | 1 + .../14/6492b04efe0aae2b8288c5c0aef6a951030fde | Bin 0 -> 170 bytes .../1d/3d6744266642cb7623e2c678c33c77b075c49f | Bin 0 -> 84 bytes .../8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba | Bin 0 -> 51 bytes .../basic-git/DOTgit/refs/heads/master | 2 +- .../test-fixtures/basic-git/subdir/sub.tf | 0 .../basic-subdir/foo/sub/baz/main.tf | 0 .../basic-subdir/foo/sub/main.tf | 3 ++ .../module/test-fixtures/basic-subdir/main.tf | 3 ++ .../module/test-fixtures/basic/subdir/sub.tf | 0 config/module/tree.go | 23 ++++----- config/module/tree_test.go | 44 ++++++++++++++++++ website/.DS_Store | Bin 6148 -> 0 bytes website/source/.DS_Store | Bin 6148 -> 0 bytes website/source/images/.DS_Store | Bin 6148 -> 0 bytes 21 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba create mode 100644 config/module/test-fixtures/basic-git/subdir/sub.tf create mode 100644 config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf create mode 100644 config/module/test-fixtures/basic-subdir/foo/sub/main.tf create mode 100644 config/module/test-fixtures/basic-subdir/main.tf create mode 100644 config/module/test-fixtures/basic/subdir/sub.tf delete mode 100644 website/.DS_Store delete mode 100644 website/source/.DS_Store delete mode 100644 website/source/images/.DS_Store diff --git a/config/module/get.go b/config/module/get.go index 89a143604..673708806 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -6,6 +6,7 @@ import ( "net/url" "os/exec" "regexp" + "strings" "syscall" ) @@ -96,6 +97,29 @@ func getRunCommand(cmd *exec.Cmd) error { return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) } +// getDirSubdir takes a source and returns a tuple of the URL without +// the subdir and the URL with the subdir. +func getDirSubdir(src string) (string, string) { + // First see if we even have an explicit subdir + idx := strings.Index(src, "//") + if idx == -1 { + return src, "" + } + + subdir := src[idx+2:] + src = src[:idx] + + // Next, check if we have query parameters and push them onto the + // URL. + if idx = strings.Index(subdir, "?"); idx > -1 { + query := subdir[idx:] + subdir = subdir[:idx] + src += query + } + + return src, subdir +} + // getForcedGetter takes a source and returns the tuple of the forced // getter and the raw URL (without the force syntax). func getForcedGetter(src string) (string, string) { diff --git a/config/module/get_test.go b/config/module/get_test.go index 742e9d794..a57d77b3a 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -45,3 +45,37 @@ func TestGet_fileForced(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestGetDirSubdir(t *testing.T) { + cases := []struct { + Input string + Dir, Sub string + }{ + { + "hashicorp.com", + "hashicorp.com", "", + }, + { + "hashicorp.com//foo", + "hashicorp.com", "foo", + }, + { + "hashicorp.com//foo?bar=baz", + "hashicorp.com?bar=baz", "foo", + }, + { + "hashicorp.com", + "hashicorp.com", "", + }, + } + + for i, tc := range cases { + adir, asub := getDirSubdir(tc.Input) + if adir != tc.Dir { + t.Fatalf("%d: bad dir: %#v", i, adir) + } + if asub != tc.Sub { + t.Fatalf("%d: bad sub: %#v", i, asub) + } + } +} diff --git a/config/module/storage.go b/config/module/storage.go index 14b5181e5..dcb0cc57c 100644 --- a/config/module/storage.go +++ b/config/module/storage.go @@ -11,3 +11,15 @@ type Storage interface { // Get will download and optionally update the given module. Get(string, bool) error } + +func getStorage(s Storage, src string, mode GetMode) (string, bool, error) { + // Get the module with the level specified if we were told to. + if mode > GetModeNone { + if err := s.Get(src, mode == GetModeUpdate); err != nil { + return "", false, err + } + } + + // Get the directory where the module is. + return s.Dir(src) +} diff --git a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG index d13fed6c9..a580d5737 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG +++ b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG @@ -1,7 +1,7 @@ -Branch +add subdir # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. -# On branch test-branch +# On branch master # Changes to be committed: -# new file: main_branch.tf +# new file: subdir/sub.tf # diff --git a/config/module/test-fixtures/basic-git/DOTgit/index b/config/module/test-fixtures/basic-git/DOTgit/index index 071f9d4396b0a2668a6251fb2c4e562646072487..99b358e271f626bc5ee9ad355678185228a62198 100644 GIT binary patch delta 144 zcmd1U!KmWm8RX2sz`z8;Aqs9GK$-zYGcqu+FvOU3OjI#Q2~oZ60hWW(V7X+yKMV|w zOBfg!zXIh%K=RM#KAOGh;x%XQZq0S&Q8x}f=?R|5z*}6Jl#*Gb52W-;(m=`>t~^Ux W{Osh`?^g^z~W9 diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD index 40709bc8e..b76ad5127 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD @@ -4,3 +4,4 @@ 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886909 -0700 checkout: moving from master to test-branch 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 Mitchell Hashimoto 1410886913 -0700 commit: Branch 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886916 -0700 checkout: moving from test-branch to master +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto 1411767116 -0700 commit: add subdir diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master index 396932ba1..f30b1d9d3 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master @@ -1,3 +1,4 @@ 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit 497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto 1410886526 -0700 commit: tag1 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886536 -0700 commit: remove tag1 +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto 1411767116 -0700 commit: add subdir diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde b/config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde new file mode 100644 index 0000000000000000000000000000000000000000..2a713ec7cf4938e12e84fc7f0aa6231c45c9a946 GIT binary patch literal 170 zcmV;b09F5Z0j^8 zP`O@wiXbUzWpDU7*Rk7Mo3iS**>8TVt&chE=W*a|=k?F7_s(rQ%jjPgA^;%)p#5;< YoH_I;rvA&A;Zp45nM>Qv3xnxYMYiKtt^fc4 literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f b/config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f new file mode 100644 index 0000000000000000000000000000000000000000..2518fd6ac51a5a3b76ebf290ea8866b81746de0c GIT binary patch literal 84 zcmV-a0IUCa0V^p=O;s?nU@$Z=Ff%bx$W6@5(<@11urNq2jQC!%i0{sU{W8An8}_#! qu{ALO0)^tzq?F7eh90TPQ}Q#mUEq)YdhE=+UK>u`%ew&du^oyiI4LLq literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba b/config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba new file mode 100644 index 0000000000000000000000000000000000000000..656ae8e33e7cb6dd2220cec705ebf707caad03c0 GIT binary patch literal 51 zcmbc HOYk88vc(gk literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master index 91c51fe57..36257b472 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master +++ b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master @@ -1 +1 @@ -1f31e97f053caeb5d6b7bffa3faf82941c99efa2 +146492b04efe0aae2b8288c5c0aef6a951030fde diff --git a/config/module/test-fixtures/basic-git/subdir/sub.tf b/config/module/test-fixtures/basic-git/subdir/sub.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf b/config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-subdir/foo/sub/main.tf b/config/module/test-fixtures/basic-subdir/foo/sub/main.tf new file mode 100644 index 000000000..22905dd53 --- /dev/null +++ b/config/module/test-fixtures/basic-subdir/foo/sub/main.tf @@ -0,0 +1,3 @@ +module "bar" { + source = "./baz" +} diff --git a/config/module/test-fixtures/basic-subdir/main.tf b/config/module/test-fixtures/basic-subdir/main.tf new file mode 100644 index 000000000..19fb5dde7 --- /dev/null +++ b/config/module/test-fixtures/basic-subdir/main.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "./foo//sub" +} diff --git a/config/module/test-fixtures/basic/subdir/sub.tf b/config/module/test-fixtures/basic/subdir/sub.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/tree.go b/config/module/tree.go index 724b59ef7..27fdecf6e 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -2,6 +2,7 @@ package module import ( "bufio" + "path/filepath" "bytes" "fmt" "strings" @@ -131,27 +132,22 @@ func (t *Tree) Load(s Storage, mode GetMode) error { children := make(map[string]*Tree) // Go through all the modules and get the directory for them. - update := mode == GetModeUpdate for _, m := range modules { if _, ok := children[m.Name]; ok { return fmt.Errorf( "module %s: duplicated. module names must be unique", m.Name) } - source, err := Detect(m.Source, t.config.Dir) + // Split out the subdir if we have one + source, subDir := getDirSubdir(m.Source) + + source, err := Detect(source, t.config.Dir) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) } - if mode > GetModeNone { - // Get the module since we specified we should - if err := s.Get(source, update); err != nil { - return err - } - } - // Get the directory where this module is so we can load it - dir, ok, err := s.Dir(source) + dir, ok, err := getStorage(s, source, mode) if err != nil { return err } @@ -160,7 +156,12 @@ func (t *Tree) Load(s Storage, mode GetMode) error { "module %s: not found, may need to be downloaded", m.Name) } - // Load the configuration + // If we have a subdirectory, then merge that in + if subDir != "" { + dir = filepath.Join(dir, subDir) + } + + // Load the configurations.Dir(source) children[m.Name], err = NewTreeModule(m.Name, dir) if err != nil { return fmt.Errorf( diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 611d451a5..837519d27 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -58,6 +58,44 @@ func TestTreeLoad_duplicate(t *testing.T) { } } +func TestTreeLoad_subdir(t *testing.T) { + storage := testStorage(t) + tree := NewTree("", testConfig(t, "basic-subdir")) + + if tree.Loaded() { + t.Fatal("should not be loaded") + } + + // This should error because we haven't gotten things yet + if err := tree.Load(storage, GetModeNone); err == nil { + t.Fatal("should error") + } + + if tree.Loaded() { + t.Fatal("should not be loaded") + } + + // This should get things + if err := tree.Load(storage, GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if !tree.Loaded() { + t.Fatal("should be loaded") + } + + // This should no longer error + if err := tree.Load(storage, GetModeNone); err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(tree.String()) + expected := strings.TrimSpace(treeLoadSubdirStr) + if actual != expected { + t.Fatalf("bad: \n\n%s", actual) + } +} + func TestTreeModules(t *testing.T) { tree := NewTree("", testConfig(t, "basic")) actual := tree.Modules() @@ -164,3 +202,9 @@ const treeLoadStr = ` root foo ` + +const treeLoadSubdirStr = ` +root + foo + bar +` diff --git a/website/.DS_Store b/website/.DS_Store deleted file mode 100644 index 1ace5ab4da7400dde76a0f96db6552f673b6cf8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-O8rh18}Ab8oU&_n&L$5{0!6e<-_sfh`6p(LeAEm8_O`xyS8dV$rL zRMq?KOFwDU?YT-jID^oN8wNEQOHVdsa&G1EsoV3tUTI^!Al8>x`UTM|m5K$ix>D-* z?FY+RSX|rOJ!thWF0Za{Ztw0NpY%y^Ode%M!(e?kQ> zfEf4<19(3OP(;UIs!?4XFsK#)Fb8fWU}Gd}bi4r#9CAouB*v&p|XJ28e-?Vu0ti ztJ`Hr$<)@NJP)b_`~nR?$6%@vJRtNVplG0h82D2L F-T=#{U%mhU diff --git a/website/source/.DS_Store b/website/source/.DS_Store deleted file mode 100644 index 86e8c58302f9bd59bafdba4d9cab5036142ede02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z@v)~Sj;i&j zQ%&5Y9hcnJftpD_@Qht?llr^86Ge@_?RvX1=+x|)%_vM<83wHnp@(}i^rKEGYKMM& zSk?RONk6IA?Ab~yID^oN8wR^FmY!_Lb!4j;z{XwzV}wD+V5$+`fN&iOs6)AFF}Mx~zcA|@ zgQ-Rx&bVnlxcM?S9ST=phy8_G&bXtIMq+>%_{_kFp4wRdw|?&bKL^o_7$63QiUFQ2 zSIZ?x$<)??(T$gI>hk&6JO`=-`~nR?$6%@vJRtNVplG0h82D2L F-T?K4U$X!J diff --git a/website/source/images/.DS_Store b/website/source/images/.DS_Store deleted file mode 100644 index 3c795397d81143926387b4ff2f13f52c222c9930..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Zgnn^$Kj9qb)=7)U{MXjOjdNmpJ8ur3&6eg|=gU+YazmuV`-i7zy$c)*N14|!9AAbO8N>iFKn#p61Lg>{X2-TdDvTH) z27b!`o(}>P(J`26R7VF4ss#Ydf?EmL*h^rJFz6UeHNpxA*QtOym75lW>vZr76XzIA zHR^Q6P4mIcm$~UsxcWN0U#M`#9gQ>+1H`~r1}5~>#`}No=l=g|63vJKVqmNo;MsDu zT!NC!-8xbn-n9bs6ch#HQjNnDFw|8Hv3M0PfhqyNKm*V*m}&$M2>l2s8fYK}{*-}t Di1A+q From 62ec0a3b4aa89cb7519cf784c34fc08148c4d144 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 26 Sep 2014 14:58:41 -0700 Subject: [PATCH 2/4] website: update website for subdirs --- website/source/docs/modules/sources.html.markdown | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/website/source/docs/modules/sources.html.markdown b/website/source/docs/modules/sources.html.markdown index fe91eb028..9ea86b699 100644 --- a/website/source/docs/modules/sources.html.markdown +++ b/website/source/docs/modules/sources.html.markdown @@ -64,10 +64,14 @@ Subdirectories within the repository can also be referenced: ``` module "consul" { - source = "github.com/hashicorp/example/subdir" + source = "github.com/hashicorp/example//subdir" } ``` +**Note:** The double-slash is important. It is what tells Terraform that +that is the separator for a subdirectory, and not part of the repository +itself. + GitHub source URLs will require that Git is installed on your system and that you have the proper access to the repository. @@ -90,10 +94,14 @@ Subdirectories within the repository can also be referenced: ``` module "consul" { - source = "bitbucket.org/hashicorp/example/subdir" + source = "bitbucket.org/hashicorp/example//subdir" } ``` +**Note:** The double-slash is important. It is what tells Terraform that +that is the separator for a subdirectory, and not part of the repository +itself. + BitBucket URLs will require that Git or Mercurial is installed on your system, depending on the source URL. From 21d90dcf4fd305a9b5179d7e4660bab0b1942c18 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 26 Sep 2014 15:22:26 -0700 Subject: [PATCH 3/4] config/module: HTTP supports subdirs --- config/module/copy_dir.go | 51 ++++++++++++++++++++++++++++++++++ config/module/get.go | 10 ++++++- config/module/get_http.go | 49 ++++++++++++++++++++++++++++++-- config/module/get_http_test.go | 29 +++++++++++++++++++ config/module/get_test.go | 4 +-- 5 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 config/module/copy_dir.go diff --git a/config/module/copy_dir.go b/config/module/copy_dir.go new file mode 100644 index 000000000..6d4cb8201 --- /dev/null +++ b/config/module/copy_dir.go @@ -0,0 +1,51 @@ +package module + +import ( + "io" + "os" + "path/filepath" +) + +// copyDir copies the src directory contents into dst. Both directories +// should already exist. +func copyDir(dst, src string) error { + walkFn := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + dstPath := filepath.Join(dst, filepath.Base(path)) + + // If we have a directory, make that subdirectory, then continue + // the walk. + if info.IsDir() { + if err := os.MkdirAll(dstPath, 0755); err != nil { + return err + } + + return copyDir(dstPath, path) + } + + // If we have a file, copy the contents. + srcF, err := os.Open(path) + if err != nil { + return err + } + defer srcF.Close() + + dstF, err := os.Create(dstPath) + if err != nil { + return err + } + defer dstF.Close() + + if _, err := io.Copy(dstF, srcF); err != nil { + return err + } + + // Chmod it + return os.Chmod(dstPath, info.Mode()) + } + + return filepath.Walk(src, walkFn) +} diff --git a/config/module/get.go b/config/module/get.go index 673708806..cd553f267 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -100,12 +100,20 @@ func getRunCommand(cmd *exec.Cmd) error { // getDirSubdir takes a source and returns a tuple of the URL without // the subdir and the URL with the subdir. func getDirSubdir(src string) (string, string) { + // Calcaulate an offset to avoid accidentally marking the scheme + // as the dir. + var offset int + if idx := strings.Index(src, "://"); idx > -1 { + offset = idx + 3 + } + // First see if we even have an explicit subdir - idx := strings.Index(src, "//") + idx := strings.Index(src[offset:], "//") if idx == -1 { return src, "" } + idx += offset subdir := src[idx+2:] src = src[:idx] diff --git a/config/module/get_http.go b/config/module/get_http.go index 7aa7d9abf..be65d921a 100644 --- a/config/module/get_http.go +++ b/config/module/get_http.go @@ -4,8 +4,11 @@ import ( "encoding/xml" "fmt" "io" + "io/ioutil" "net/http" "net/url" + "os" + "path/filepath" "strings" ) @@ -62,8 +65,50 @@ func (g *HttpGetter) Get(dst string, u *url.URL) error { return fmt.Errorf("no source URL was returned") } - // Get it! - return Get(dst, source) + // If there is a subdir component, then we download the root separately + // into a temporary directory, then copy over the proper subdir. + source, subDir := getDirSubdir(source) + if subDir == "" { + return Get(dst, source) + } + + // We have a subdir, time to jump some hoops + return g.getSubdir(dst, source, subDir) +} + +// getSubdir downloads the source into the destination, but with +// the proper subdir. +func (g *HttpGetter) getSubdir(dst, source, subDir string) error { + // Create a temporary directory to store the full source + td, err := ioutil.TempDir("", "tf") + if err != nil { + return err + } + defer os.RemoveAll(td) + + // Download that into the given directory + if err := Get(td, source); err != nil { + return err + } + + // Make sure the subdir path actually exists + sourcePath := filepath.Join(td, subDir) + if _, err := os.Stat(sourcePath); err != nil { + return fmt.Errorf( + "Error downloading %s: %s", source, err) + } + + // Copy the subdirectory into our actual destination. + if err := os.RemoveAll(dst); err != nil { + return err + } + + // Make the final destination + if err := os.MkdirAll(dst, 0755); err != nil { + return err + } + + return copyDir(dst, sourcePath) } // parseMeta looks for the first meta tag in the given reader that diff --git a/config/module/get_http_test.go b/config/module/get_http_test.go index 8fa61d2e4..5eb83a619 100644 --- a/config/module/get_http_test.go +++ b/config/module/get_http_test.go @@ -62,6 +62,30 @@ func TestHttpGetter_meta(t *testing.T) { } } +func TestHttpGetter_metaSubdir(t *testing.T) { + ln := testHttpServer(t) + defer ln.Close() + + g := new(HttpGetter) + dst := tempDir(t) + + var u url.URL + u.Scheme = "http" + u.Host = ln.Addr().String() + u.Path = "/meta-subdir" + + // Get it! + if err := g.Get(dst, &u); err != nil { + t.Fatalf("err: %s", err) + } + + // Verify the main file exists + mainPath := filepath.Join(dst, "sub.tf") + if _, err := os.Stat(mainPath); err != nil { + t.Fatalf("err: %s", err) + } +} + func TestHttpGetter_none(t *testing.T) { ln := testHttpServer(t) defer ln.Close() @@ -89,6 +113,7 @@ func testHttpServer(t *testing.T) net.Listener { mux := http.NewServeMux() mux.HandleFunc("/header", testHttpHandlerHeader) mux.HandleFunc("/meta", testHttpHandlerMeta) + mux.HandleFunc("/meta-subdir", testHttpHandlerMetaSubdir) var server http.Server server.Handler = mux @@ -106,6 +131,10 @@ func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) { w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String()))) } +func testHttpHandlerMetaSubdir(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//subdir").String()))) +} + func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) { w.Write([]byte(testHttpNoneStr)) } diff --git a/config/module/get_test.go b/config/module/get_test.go index a57d77b3a..85488577d 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -64,8 +64,8 @@ func TestGetDirSubdir(t *testing.T) { "hashicorp.com?bar=baz", "foo", }, { - "hashicorp.com", - "hashicorp.com", "", + "file://foo//bar", + "file://foo", "bar", }, } From 4fbd5abc63bee66bd61ef06b0777ecc6b36746a4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 26 Sep 2014 15:30:36 -0700 Subject: [PATCH 4/4] config/module: support GitHub paths without // --- config/module/detect_github.go | 12 +++++++++++- config/module/detect_github_test.go | 8 ++++++++ config/module/tree.go | 6 ++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/config/module/detect_github.go b/config/module/detect_github.go index e4854cf42..7e7deb653 100644 --- a/config/module/detect_github.go +++ b/config/module/detect_github.go @@ -25,7 +25,13 @@ func (d *GitHubDetector) Detect(src, _ string) (string, bool, error) { } func (d *GitHubDetector) detectHTTP(src string) (string, bool, error) { - urlStr := fmt.Sprintf("https://%s", src) + parts := strings.Split(src, "/") + if len(parts) < 3 { + return "", false, fmt.Errorf( + "GitHub URLs should be github.com/username/repo") + } + + urlStr := fmt.Sprintf("https://%s", strings.Join(parts[:3], "/")) url, err := url.Parse(urlStr) if err != nil { return "", true, fmt.Errorf("error parsing GitHub URL: %s", err) @@ -35,6 +41,10 @@ func (d *GitHubDetector) detectHTTP(src string) (string, bool, error) { url.Path += ".git" } + if len(parts) > 3 { + url.Path += "//" + strings.Join(parts[3:], "/") + } + return "git::" + url.String(), true, nil } diff --git a/config/module/detect_github_test.go b/config/module/detect_github_test.go index 37fac84c8..822e1806d 100644 --- a/config/module/detect_github_test.go +++ b/config/module/detect_github_test.go @@ -12,6 +12,10 @@ func TestGitHubDetector(t *testing.T) { // HTTP {"github.com/hashicorp/foo", "git::https://github.com/hashicorp/foo.git"}, {"github.com/hashicorp/foo.git", "git::https://github.com/hashicorp/foo.git"}, + { + "github.com/hashicorp/foo/bar", + "git::https://github.com/hashicorp/foo.git//bar", + }, { "github.com/hashicorp/foo?foo=bar", "git::https://github.com/hashicorp/foo.git?foo=bar", @@ -23,6 +27,10 @@ func TestGitHubDetector(t *testing.T) { // SSH {"git@github.com:hashicorp/foo.git", "git::ssh://git@github.com/hashicorp/foo.git"}, + { + "git@github.com:hashicorp/foo.git//bar", + "git::ssh://git@github.com/hashicorp/foo.git//bar", + }, { "git@github.com:hashicorp/foo.git?foo=bar", "git::ssh://git@github.com/hashicorp/foo.git?foo=bar", diff --git a/config/module/tree.go b/config/module/tree.go index 27fdecf6e..b7cfc5108 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -146,6 +146,12 @@ func (t *Tree) Load(s Storage, mode GetMode) error { return fmt.Errorf("module %s: %s", m.Name, err) } + // Check if the detector introduced something new. + source, subDir2 := getDirSubdir(source) + if subDir2 != "" { + subDir = filepath.Join(subDir2, subDir) + } + // Get the directory where this module is so we can load it dir, ok, err := getStorage(s, source, mode) if err != nil {