From 7aa30fdf25d6456343a725d13ae40c3775f61735 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Fri, 26 Sep 2014 14:56:04 -0700 Subject: [PATCH] config: support subdirectories with the "//" syntax --- config/module/get.go | 24 ++++++++++ config/module/get_test.go | 34 ++++++++++++++ config/module/storage.go | 12 +++++ .../basic-git/DOTgit/COMMIT_EDITMSG | 6 +-- .../test-fixtures/basic-git/DOTgit/index | Bin 104 -> 184 bytes .../test-fixtures/basic-git/DOTgit/logs/HEAD | 1 + .../basic-git/DOTgit/logs/refs/heads/master | 1 + .../14/6492b04efe0aae2b8288c5c0aef6a951030fde | Bin 0 -> 170 bytes .../1d/3d6744266642cb7623e2c678c33c77b075c49f | Bin 0 -> 84 bytes .../8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba | Bin 0 -> 51 bytes .../basic-git/DOTgit/refs/heads/master | 2 +- .../test-fixtures/basic-git/subdir/sub.tf | 0 .../basic-subdir/foo/sub/baz/main.tf | 0 .../basic-subdir/foo/sub/main.tf | 3 ++ .../module/test-fixtures/basic-subdir/main.tf | 3 ++ .../module/test-fixtures/basic/subdir/sub.tf | 0 config/module/tree.go | 23 ++++----- config/module/tree_test.go | 44 ++++++++++++++++++ website/.DS_Store | Bin 6148 -> 0 bytes website/source/.DS_Store | Bin 6148 -> 0 bytes website/source/images/.DS_Store | Bin 6148 -> 0 bytes 21 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f create mode 100644 config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba create mode 100644 config/module/test-fixtures/basic-git/subdir/sub.tf create mode 100644 config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf create mode 100644 config/module/test-fixtures/basic-subdir/foo/sub/main.tf create mode 100644 config/module/test-fixtures/basic-subdir/main.tf create mode 100644 config/module/test-fixtures/basic/subdir/sub.tf delete mode 100644 website/.DS_Store delete mode 100644 website/source/.DS_Store delete mode 100644 website/source/images/.DS_Store diff --git a/config/module/get.go b/config/module/get.go index 89a143604..673708806 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -6,6 +6,7 @@ import ( "net/url" "os/exec" "regexp" + "strings" "syscall" ) @@ -96,6 +97,29 @@ func getRunCommand(cmd *exec.Cmd) error { return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) } +// getDirSubdir takes a source and returns a tuple of the URL without +// the subdir and the URL with the subdir. +func getDirSubdir(src string) (string, string) { + // First see if we even have an explicit subdir + idx := strings.Index(src, "//") + if idx == -1 { + return src, "" + } + + subdir := src[idx+2:] + src = src[:idx] + + // Next, check if we have query parameters and push them onto the + // URL. + if idx = strings.Index(subdir, "?"); idx > -1 { + query := subdir[idx:] + subdir = subdir[:idx] + src += query + } + + return src, subdir +} + // getForcedGetter takes a source and returns the tuple of the forced // getter and the raw URL (without the force syntax). func getForcedGetter(src string) (string, string) { diff --git a/config/module/get_test.go b/config/module/get_test.go index 742e9d794..a57d77b3a 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -45,3 +45,37 @@ func TestGet_fileForced(t *testing.T) { t.Fatalf("err: %s", err) } } + +func TestGetDirSubdir(t *testing.T) { + cases := []struct { + Input string + Dir, Sub string + }{ + { + "hashicorp.com", + "hashicorp.com", "", + }, + { + "hashicorp.com//foo", + "hashicorp.com", "foo", + }, + { + "hashicorp.com//foo?bar=baz", + "hashicorp.com?bar=baz", "foo", + }, + { + "hashicorp.com", + "hashicorp.com", "", + }, + } + + for i, tc := range cases { + adir, asub := getDirSubdir(tc.Input) + if adir != tc.Dir { + t.Fatalf("%d: bad dir: %#v", i, adir) + } + if asub != tc.Sub { + t.Fatalf("%d: bad sub: %#v", i, asub) + } + } +} diff --git a/config/module/storage.go b/config/module/storage.go index 14b5181e5..dcb0cc57c 100644 --- a/config/module/storage.go +++ b/config/module/storage.go @@ -11,3 +11,15 @@ type Storage interface { // Get will download and optionally update the given module. Get(string, bool) error } + +func getStorage(s Storage, src string, mode GetMode) (string, bool, error) { + // Get the module with the level specified if we were told to. + if mode > GetModeNone { + if err := s.Get(src, mode == GetModeUpdate); err != nil { + return "", false, err + } + } + + // Get the directory where the module is. + return s.Dir(src) +} diff --git a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG index d13fed6c9..a580d5737 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG +++ b/config/module/test-fixtures/basic-git/DOTgit/COMMIT_EDITMSG @@ -1,7 +1,7 @@ -Branch +add subdir # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. -# On branch test-branch +# On branch master # Changes to be committed: -# new file: main_branch.tf +# new file: subdir/sub.tf # diff --git a/config/module/test-fixtures/basic-git/DOTgit/index b/config/module/test-fixtures/basic-git/DOTgit/index index 071f9d4396b0a2668a6251fb2c4e562646072487..99b358e271f626bc5ee9ad355678185228a62198 100644 GIT binary patch delta 144 zcmd1U!KmWm8RX2sz`z8;Aqs9GK$-zYGcqu+FvOU3OjI#Q2~oZ60hWW(V7X+yKMV|w zOBfg!zXIh%K=RM#KAOGh;x%XQZq0S&Q8x}f=?R|5z*}6Jl#*Gb52W-;(m=`>t~^Ux W{Osh`?^g^z~W9 diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD index 40709bc8e..b76ad5127 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/HEAD @@ -4,3 +4,4 @@ 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886909 -0700 checkout: moving from master to test-branch 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 Mitchell Hashimoto 1410886913 -0700 commit: Branch 7b7614f8759ac8b5e4b02be65ad8e2667be6dd87 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886916 -0700 checkout: moving from test-branch to master +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto 1411767116 -0700 commit: add subdir diff --git a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master index 396932ba1..f30b1d9d3 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master +++ b/config/module/test-fixtures/basic-git/DOTgit/logs/refs/heads/master @@ -1,3 +1,4 @@ 0000000000000000000000000000000000000000 497bc37401eb3c9b11865b1768725b64066eccee Mitchell Hashimoto 1410850637 -0700 commit (initial): A commit 497bc37401eb3c9b11865b1768725b64066eccee 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 Mitchell Hashimoto 1410886526 -0700 commit: tag1 243f0fc5c4e586d1a3daa54c981b6f34e9ab1085 1f31e97f053caeb5d6b7bffa3faf82941c99efa2 Mitchell Hashimoto 1410886536 -0700 commit: remove tag1 +1f31e97f053caeb5d6b7bffa3faf82941c99efa2 146492b04efe0aae2b8288c5c0aef6a951030fde Mitchell Hashimoto 1411767116 -0700 commit: add subdir diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde b/config/module/test-fixtures/basic-git/DOTgit/objects/14/6492b04efe0aae2b8288c5c0aef6a951030fde new file mode 100644 index 0000000000000000000000000000000000000000..2a713ec7cf4938e12e84fc7f0aa6231c45c9a946 GIT binary patch literal 170 zcmV;b09F5Z0j^8 zP`O@wiXbUzWpDU7*Rk7Mo3iS**>8TVt&chE=W*a|=k?F7_s(rQ%jjPgA^;%)p#5;< YoH_I;rvA&A;Zp45nM>Qv3xnxYMYiKtt^fc4 literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f b/config/module/test-fixtures/basic-git/DOTgit/objects/1d/3d6744266642cb7623e2c678c33c77b075c49f new file mode 100644 index 0000000000000000000000000000000000000000..2518fd6ac51a5a3b76ebf290ea8866b81746de0c GIT binary patch literal 84 zcmV-a0IUCa0V^p=O;s?nU@$Z=Ff%bx$W6@5(<@11urNq2jQC!%i0{sU{W8An8}_#! qu{ALO0)^tzq?F7eh90TPQ}Q#mUEq)YdhE=+UK>u`%ew&du^oyiI4LLq literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba b/config/module/test-fixtures/basic-git/DOTgit/objects/8c/1a79ca1f98b6d00f5bf5c6cc9e8d3c092dd3ba new file mode 100644 index 0000000000000000000000000000000000000000..656ae8e33e7cb6dd2220cec705ebf707caad03c0 GIT binary patch literal 51 zcmbc HOYk88vc(gk literal 0 HcmV?d00001 diff --git a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master index 91c51fe57..36257b472 100644 --- a/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master +++ b/config/module/test-fixtures/basic-git/DOTgit/refs/heads/master @@ -1 +1 @@ -1f31e97f053caeb5d6b7bffa3faf82941c99efa2 +146492b04efe0aae2b8288c5c0aef6a951030fde diff --git a/config/module/test-fixtures/basic-git/subdir/sub.tf b/config/module/test-fixtures/basic-git/subdir/sub.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf b/config/module/test-fixtures/basic-subdir/foo/sub/baz/main.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/test-fixtures/basic-subdir/foo/sub/main.tf b/config/module/test-fixtures/basic-subdir/foo/sub/main.tf new file mode 100644 index 000000000..22905dd53 --- /dev/null +++ b/config/module/test-fixtures/basic-subdir/foo/sub/main.tf @@ -0,0 +1,3 @@ +module "bar" { + source = "./baz" +} diff --git a/config/module/test-fixtures/basic-subdir/main.tf b/config/module/test-fixtures/basic-subdir/main.tf new file mode 100644 index 000000000..19fb5dde7 --- /dev/null +++ b/config/module/test-fixtures/basic-subdir/main.tf @@ -0,0 +1,3 @@ +module "foo" { + source = "./foo//sub" +} diff --git a/config/module/test-fixtures/basic/subdir/sub.tf b/config/module/test-fixtures/basic/subdir/sub.tf new file mode 100644 index 000000000..e69de29bb diff --git a/config/module/tree.go b/config/module/tree.go index 724b59ef7..27fdecf6e 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -2,6 +2,7 @@ package module import ( "bufio" + "path/filepath" "bytes" "fmt" "strings" @@ -131,27 +132,22 @@ func (t *Tree) Load(s Storage, mode GetMode) error { children := make(map[string]*Tree) // Go through all the modules and get the directory for them. - update := mode == GetModeUpdate for _, m := range modules { if _, ok := children[m.Name]; ok { return fmt.Errorf( "module %s: duplicated. module names must be unique", m.Name) } - source, err := Detect(m.Source, t.config.Dir) + // Split out the subdir if we have one + source, subDir := getDirSubdir(m.Source) + + source, err := Detect(source, t.config.Dir) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) } - if mode > GetModeNone { - // Get the module since we specified we should - if err := s.Get(source, update); err != nil { - return err - } - } - // Get the directory where this module is so we can load it - dir, ok, err := s.Dir(source) + dir, ok, err := getStorage(s, source, mode) if err != nil { return err } @@ -160,7 +156,12 @@ func (t *Tree) Load(s Storage, mode GetMode) error { "module %s: not found, may need to be downloaded", m.Name) } - // Load the configuration + // If we have a subdirectory, then merge that in + if subDir != "" { + dir = filepath.Join(dir, subDir) + } + + // Load the configurations.Dir(source) children[m.Name], err = NewTreeModule(m.Name, dir) if err != nil { return fmt.Errorf( diff --git a/config/module/tree_test.go b/config/module/tree_test.go index 611d451a5..837519d27 100644 --- a/config/module/tree_test.go +++ b/config/module/tree_test.go @@ -58,6 +58,44 @@ func TestTreeLoad_duplicate(t *testing.T) { } } +func TestTreeLoad_subdir(t *testing.T) { + storage := testStorage(t) + tree := NewTree("", testConfig(t, "basic-subdir")) + + if tree.Loaded() { + t.Fatal("should not be loaded") + } + + // This should error because we haven't gotten things yet + if err := tree.Load(storage, GetModeNone); err == nil { + t.Fatal("should error") + } + + if tree.Loaded() { + t.Fatal("should not be loaded") + } + + // This should get things + if err := tree.Load(storage, GetModeGet); err != nil { + t.Fatalf("err: %s", err) + } + + if !tree.Loaded() { + t.Fatal("should be loaded") + } + + // This should no longer error + if err := tree.Load(storage, GetModeNone); err != nil { + t.Fatalf("err: %s", err) + } + + actual := strings.TrimSpace(tree.String()) + expected := strings.TrimSpace(treeLoadSubdirStr) + if actual != expected { + t.Fatalf("bad: \n\n%s", actual) + } +} + func TestTreeModules(t *testing.T) { tree := NewTree("", testConfig(t, "basic")) actual := tree.Modules() @@ -164,3 +202,9 @@ const treeLoadStr = ` root foo ` + +const treeLoadSubdirStr = ` +root + foo + bar +` diff --git a/website/.DS_Store b/website/.DS_Store deleted file mode 100644 index 1ace5ab4da7400dde76a0f96db6552f673b6cf8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z-O8rh18}Ab8oU&_n&L$5{0!6e<-_sfh`6p(LeAEm8_O`xyS8dV$rL zRMq?KOFwDU?YT-jID^oN8wNEQOHVdsa&G1EsoV3tUTI^!Al8>x`UTM|m5K$ix>D-* z?FY+RSX|rOJ!thWF0Za{Ztw0NpY%y^Ode%M!(e?kQ> zfEf4<19(3OP(;UIs!?4XFsK#)Fb8fWU}Gd}bi4r#9CAouB*v&p|XJ28e-?Vu0ti ztJ`Hr$<)@NJP)b_`~nR?$6%@vJRtNVplG0h82D2L F-T=#{U%mhU diff --git a/website/source/.DS_Store b/website/source/.DS_Store deleted file mode 100644 index 86e8c58302f9bd59bafdba4d9cab5036142ede02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Z@v)~Sj;i&j zQ%&5Y9hcnJftpD_@Qht?llr^86Ge@_?RvX1=+x|)%_vM<83wHnp@(}i^rKEGYKMM& zSk?RONk6IA?Ab~yID^oN8wR^FmY!_Lb!4j;z{XwzV}wD+V5$+`fN&iOs6)AFF}Mx~zcA|@ zgQ-Rx&bVnlxcM?S9ST=phy8_G&bXtIMq+>%_{_kFp4wRdw|?&bKL^o_7$63QiUFQ2 zSIZ?x$<)??(T$gI>hk&6JO`=-`~nR?$6%@vJRtNVplG0h82D2L F-T?K4U$X!J diff --git a/website/source/images/.DS_Store b/website/source/images/.DS_Store deleted file mode 100644 index 3c795397d81143926387b4ff2f13f52c222c9930..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}T>S5Zgnn^$Kj9qb)=7)U{MXjOjdNmpJ8ur3&6eg|=gU+YazmuV`-i7zy$c)*N14|!9AAbO8N>iFKn#p61Lg>{X2-TdDvTH) z27b!`o(}>P(J`26R7VF4ss#Ydf?EmL*h^rJFz6UeHNpxA*QtOym75lW>vZr76XzIA zHR^Q6P4mIcm$~UsxcWN0U#M`#9gQ>+1H`~r1}5~>#`}No=l=g|63vJKVqmNo;MsDu zT!NC!-8xbn-n9bs6ch#HQjNnDFw|8Hv3M0PfhqyNKm*V*m}&$M2>l2s8fYK}{*-}t Di1A+q