From 06f4ac8166595b47638c2ae39e68a6b0e6f549bf Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 15 Oct 2015 13:36:58 -0700 Subject: [PATCH] config/module: use go-getter --- .gitignore | 1 + config/module/copy_dir.go | 76 --------- config/module/detect.go | 92 ----------- config/module/detect_bitbucket.go | 66 -------- config/module/detect_bitbucket_test.go | 67 -------- config/module/detect_file.go | 60 ------- config/module/detect_file_test.go | 88 ---------- config/module/detect_github.go | 73 --------- config/module/detect_github_test.go | 55 ------- config/module/detect_test.go | 51 ------ config/module/folder_storage.go | 65 -------- config/module/folder_storage_test.go | 48 ------ config/module/get.go | 217 +++---------------------- config/module/get_file.go | 46 ------ config/module/get_file_test.go | 104 ------------ config/module/get_git.go | 74 --------- config/module/get_git_test.go | 143 ---------------- config/module/get_hg.go | 89 ---------- config/module/get_hg_test.go | 81 --------- config/module/get_http.go | 173 -------------------- config/module/get_http_test.go | 155 ------------------ config/module/get_test.go | 128 --------------- config/module/module_test.go | 25 +-- config/module/storage.go | 25 --- config/module/tree.go | 28 +--- 25 files changed, 32 insertions(+), 1998 deletions(-) delete mode 100644 config/module/copy_dir.go delete mode 100644 config/module/detect.go delete mode 100644 config/module/detect_bitbucket.go delete mode 100644 config/module/detect_bitbucket_test.go delete mode 100644 config/module/detect_file.go delete mode 100644 config/module/detect_file_test.go delete mode 100644 config/module/detect_github.go delete mode 100644 config/module/detect_github_test.go delete mode 100644 config/module/detect_test.go delete mode 100644 config/module/folder_storage.go delete mode 100644 config/module/folder_storage_test.go delete mode 100644 config/module/get_file.go delete mode 100644 config/module/get_file_test.go delete mode 100644 config/module/get_git.go delete mode 100644 config/module/get_git_test.go delete mode 100644 config/module/get_hg.go delete mode 100644 config/module/get_hg_test.go delete mode 100644 config/module/get_http.go delete mode 100644 config/module/get_http_test.go delete mode 100644 config/module/get_test.go delete mode 100644 config/module/storage.go diff --git a/.gitignore b/.gitignore index 66ea31701..5a230d5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.dll *.exe +.DS_Store example.tf terraform.tfplan terraform.tfstate diff --git a/config/module/copy_dir.go b/config/module/copy_dir.go deleted file mode 100644 index f2ae63b77..000000000 --- a/config/module/copy_dir.go +++ /dev/null @@ -1,76 +0,0 @@ -package module - -import ( - "io" - "os" - "path/filepath" - "strings" -) - -// copyDir copies the src directory contents into dst. Both directories -// should already exist. -func copyDir(dst, src string) error { - src, err := filepath.EvalSymlinks(src) - if err != nil { - return err - } - - walkFn := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if path == src { - return nil - } - - if strings.HasPrefix(filepath.Base(path), ".") { - // Skip any dot files - if info.IsDir() { - return filepath.SkipDir - } else { - return nil - } - } - - // The "path" has the src prefixed to it. We need to join our - // destination with the path without the src on it. - dstPath := filepath.Join(dst, path[len(src):]) - - // If we have a directory, make that subdirectory, then continue - // the walk. - if info.IsDir() { - if path == filepath.Join(src, dst) { - // dst is in src; don't walk it. - return nil - } - - if err := os.MkdirAll(dstPath, 0755); err != nil { - return err - } - - return nil - } - - // If we have a file, copy the contents. - srcF, err := os.Open(path) - if err != nil { - return err - } - defer srcF.Close() - - dstF, err := os.Create(dstPath) - if err != nil { - return err - } - defer dstF.Close() - - if _, err := io.Copy(dstF, srcF); err != nil { - return err - } - - // Chmod it - return os.Chmod(dstPath, info.Mode()) - } - - return filepath.Walk(src, walkFn) -} diff --git a/config/module/detect.go b/config/module/detect.go deleted file mode 100644 index 51e07f725..000000000 --- a/config/module/detect.go +++ /dev/null @@ -1,92 +0,0 @@ -package module - -import ( - "fmt" - "path/filepath" - - "github.com/hashicorp/terraform/helper/url" -) - -// Detector defines the interface that an invalid URL or a URL with a blank -// scheme is passed through in order to determine if its shorthand for -// something else well-known. -type Detector interface { - // Detect will detect whether the string matches a known pattern to - // turn it into a proper URL. - Detect(string, string) (string, bool, error) -} - -// Detectors is the list of detectors that are tried on an invalid URL. -// This is also the order they're tried (index 0 is first). -var Detectors []Detector - -func init() { - Detectors = []Detector{ - new(GitHubDetector), - new(BitBucketDetector), - new(FileDetector), - } -} - -// Detect turns a source string into another source string if it is -// detected to be of a known pattern. -// -// This is safe to be called with an already valid source string: Detect -// will just return it. -func Detect(src string, pwd string) (string, error) { - getForce, getSrc := getForcedGetter(src) - - // Separate out the subdir if there is one, we don't pass that to detect - getSrc, subDir := getDirSubdir(getSrc) - - u, err := url.Parse(getSrc) - if err == nil && u.Scheme != "" { - // Valid URL - return src, nil - } - - for _, d := range Detectors { - result, ok, err := d.Detect(getSrc, pwd) - if err != nil { - return "", err - } - if !ok { - continue - } - - var detectForce string - detectForce, result = getForcedGetter(result) - result, detectSubdir := getDirSubdir(result) - - // If we have a subdir from the detection, then prepend it to our - // requested subdir. - if detectSubdir != "" { - if subDir != "" { - subDir = filepath.Join(detectSubdir, subDir) - } else { - subDir = detectSubdir - } - } - if subDir != "" { - u, err := url.Parse(result) - if err != nil { - return "", fmt.Errorf("Error parsing URL: %s", err) - } - u.Path += "//" + subDir - result = u.String() - } - - // Preserve the forced getter if it exists. We try to use the - // original set force first, followed by any force set by the - // detector. - if getForce != "" { - result = fmt.Sprintf("%s::%s", getForce, result) - } else if detectForce != "" { - result = fmt.Sprintf("%s::%s", detectForce, result) - } - - return result, nil - } - - return "", fmt.Errorf("invalid source string: %s", src) -} diff --git a/config/module/detect_bitbucket.go b/config/module/detect_bitbucket.go deleted file mode 100644 index 657637c09..000000000 --- a/config/module/detect_bitbucket.go +++ /dev/null @@ -1,66 +0,0 @@ -package module - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" -) - -// BitBucketDetector implements Detector to detect BitBucket URLs and turn -// them into URLs that the Git or Hg Getter can understand. -type BitBucketDetector struct{} - -func (d *BitBucketDetector) Detect(src, _ string) (string, bool, error) { - if len(src) == 0 { - return "", false, nil - } - - if strings.HasPrefix(src, "bitbucket.org/") { - return d.detectHTTP(src) - } - - return "", false, nil -} - -func (d *BitBucketDetector) detectHTTP(src string) (string, bool, error) { - u, err := url.Parse("https://" + src) - if err != nil { - return "", true, fmt.Errorf("error parsing BitBucket URL: %s", err) - } - - // We need to get info on this BitBucket repository to determine whether - // it is Git or Hg. - var info struct { - SCM string `json:"scm"` - } - infoUrl := "https://api.bitbucket.org/1.0/repositories" + u.Path - resp, err := http.Get(infoUrl) - if err != nil { - return "", true, fmt.Errorf("error looking up BitBucket URL: %s", err) - } - if resp.StatusCode == 403 { - // A private repo - return "", true, fmt.Errorf( - "shorthand BitBucket URL can't be used for private repos, " + - "please use a full URL") - } - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&info); err != nil { - return "", true, fmt.Errorf("error looking up BitBucket URL: %s", err) - } - - switch info.SCM { - case "git": - if !strings.HasSuffix(u.Path, ".git") { - u.Path += ".git" - } - - return "git::" + u.String(), true, nil - case "hg": - return "hg::" + u.String(), true, nil - default: - return "", true, fmt.Errorf("unknown BitBucket SCM type: %s", info.SCM) - } -} diff --git a/config/module/detect_bitbucket_test.go b/config/module/detect_bitbucket_test.go deleted file mode 100644 index b05fd5999..000000000 --- a/config/module/detect_bitbucket_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package module - -import ( - "net/http" - "strings" - "testing" -) - -const testBBUrl = "https://bitbucket.org/hashicorp/tf-test-git" - -func TestBitBucketDetector(t *testing.T) { - t.Parallel() - - if _, err := http.Get(testBBUrl); err != nil { - t.Log("internet may not be working, skipping BB tests") - t.Skip() - } - - cases := []struct { - Input string - Output string - }{ - // HTTP - { - "bitbucket.org/hashicorp/tf-test-git", - "git::https://bitbucket.org/hashicorp/tf-test-git.git", - }, - { - "bitbucket.org/hashicorp/tf-test-git.git", - "git::https://bitbucket.org/hashicorp/tf-test-git.git", - }, - { - "bitbucket.org/hashicorp/tf-test-hg", - "hg::https://bitbucket.org/hashicorp/tf-test-hg", - }, - } - - pwd := "/pwd" - f := new(BitBucketDetector) - for i, tc := range cases { - var err error - for i := 0; i < 3; i++ { - var output string - var ok bool - output, ok, err = f.Detect(tc.Input, pwd) - if err != nil { - if strings.Contains(err.Error(), "invalid character") { - continue - } - - t.Fatalf("err: %s", err) - } - if !ok { - t.Fatal("not ok") - } - - if output != tc.Output { - t.Fatalf("%d: bad: %#v", i, output) - } - - break - } - if i >= 3 { - t.Fatalf("failure from bitbucket: %s", err) - } - } -} diff --git a/config/module/detect_file.go b/config/module/detect_file.go deleted file mode 100644 index 859739f95..000000000 --- a/config/module/detect_file.go +++ /dev/null @@ -1,60 +0,0 @@ -package module - -import ( - "fmt" - "os" - "path/filepath" - "runtime" -) - -// FileDetector implements Detector to detect file paths. -type FileDetector struct{} - -func (d *FileDetector) Detect(src, pwd string) (string, bool, error) { - if len(src) == 0 { - return "", false, nil - } - - if !filepath.IsAbs(src) { - if pwd == "" { - return "", true, fmt.Errorf( - "relative paths require a module with a pwd") - } - - // Stat the pwd to determine if its a symbolic link. If it is, - // then the pwd becomes the original directory. Otherwise, - // `filepath.Join` below does some weird stuff. - // - // We just ignore if the pwd doesn't exist. That error will be - // caught later when we try to use the URL. - if fi, err := os.Lstat(pwd); !os.IsNotExist(err) { - if err != nil { - return "", true, err - } - if fi.Mode()&os.ModeSymlink != 0 { - pwd, err = os.Readlink(pwd) - if err != nil { - return "", true, err - } - } - } - - src = filepath.Join(pwd, src) - } - - return fmtFileURL(src), true, nil -} - -func fmtFileURL(path string) string { - if runtime.GOOS == "windows" { - // Make sure we're using "/" on Windows. URLs are "/"-based. - path = filepath.ToSlash(path) - return fmt.Sprintf("file://%s", path) - } - - // Make sure that we don't start with "/" since we add that below. - if path[0] == '/' { - path = path[1:] - } - return fmt.Sprintf("file:///%s", path) -} diff --git a/config/module/detect_file_test.go b/config/module/detect_file_test.go deleted file mode 100644 index 3e9db8bba..000000000 --- a/config/module/detect_file_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package module - -import ( - "runtime" - "testing" -) - -type fileTest struct { - in, pwd, out string - err bool -} - -var fileTests = []fileTest{ - {"./foo", "/pwd", "file:///pwd/foo", false}, - {"./foo?foo=bar", "/pwd", "file:///pwd/foo?foo=bar", false}, - {"foo", "/pwd", "file:///pwd/foo", false}, -} - -var unixFileTests = []fileTest{ - {"/foo", "/pwd", "file:///foo", false}, - {"/foo?bar=baz", "/pwd", "file:///foo?bar=baz", false}, -} - -var winFileTests = []fileTest{ - {"/foo", "/pwd", "file:///pwd/foo", false}, - {`C:\`, `/pwd`, `file://C:/`, false}, - {`C:\?bar=baz`, `/pwd`, `file://C:/?bar=baz`, false}, -} - -func TestFileDetector(t *testing.T) { - if runtime.GOOS == "windows" { - fileTests = append(fileTests, winFileTests...) - } else { - fileTests = append(fileTests, unixFileTests...) - } - - f := new(FileDetector) - for i, tc := range fileTests { - out, ok, err := f.Detect(tc.in, tc.pwd) - if err != nil { - t.Fatalf("err: %s", err) - } - if !ok { - t.Fatal("not ok") - } - - if out != tc.out { - t.Fatalf("%d: bad: %#v", i, out) - } - } -} - -var noPwdFileTests = []fileTest{ - {in: "./foo", pwd: "", out: "", err: true}, - {in: "foo", pwd: "", out: "", err: true}, -} - -var noPwdUnixFileTests = []fileTest{ - {in: "/foo", pwd: "", out: "file:///foo", err: false}, -} - -var noPwdWinFileTests = []fileTest{ - {in: "/foo", pwd: "", out: "", err: true}, - {in: `C:\`, pwd: ``, out: `file://C:/`, err: false}, -} - -func TestFileDetector_noPwd(t *testing.T) { - if runtime.GOOS == "windows" { - noPwdFileTests = append(noPwdFileTests, noPwdWinFileTests...) - } else { - noPwdFileTests = append(noPwdFileTests, noPwdUnixFileTests...) - } - - f := new(FileDetector) - for i, tc := range noPwdFileTests { - out, ok, err := f.Detect(tc.in, tc.pwd) - if err != nil != tc.err { - t.Fatalf("%d: err: %s", i, err) - } - if !ok { - t.Fatal("not ok") - } - - if out != tc.out { - t.Fatalf("%d: bad: %#v", i, out) - } - } -} diff --git a/config/module/detect_github.go b/config/module/detect_github.go deleted file mode 100644 index c4a4e89f0..000000000 --- a/config/module/detect_github.go +++ /dev/null @@ -1,73 +0,0 @@ -package module - -import ( - "fmt" - "net/url" - "strings" -) - -// GitHubDetector implements Detector to detect GitHub URLs and turn -// them into URLs that the Git Getter can understand. -type GitHubDetector struct{} - -func (d *GitHubDetector) Detect(src, _ string) (string, bool, error) { - if len(src) == 0 { - return "", false, nil - } - - if strings.HasPrefix(src, "github.com/") { - return d.detectHTTP(src) - } else if strings.HasPrefix(src, "git@github.com:") { - return d.detectSSH(src) - } - - return "", false, nil -} - -func (d *GitHubDetector) detectHTTP(src string) (string, bool, error) { - parts := strings.Split(src, "/") - if len(parts) < 3 { - return "", false, fmt.Errorf( - "GitHub URLs should be github.com/username/repo") - } - - urlStr := fmt.Sprintf("https://%s", strings.Join(parts[:3], "/")) - url, err := url.Parse(urlStr) - if err != nil { - return "", true, fmt.Errorf("error parsing GitHub URL: %s", err) - } - - if !strings.HasSuffix(url.Path, ".git") { - url.Path += ".git" - } - - if len(parts) > 3 { - url.Path += "//" + strings.Join(parts[3:], "/") - } - - return "git::" + url.String(), true, nil -} - -func (d *GitHubDetector) detectSSH(src string) (string, bool, error) { - idx := strings.Index(src, ":") - qidx := strings.Index(src, "?") - if qidx == -1 { - qidx = len(src) - } - - var u url.URL - u.Scheme = "ssh" - u.User = url.User("git") - u.Host = "github.com" - u.Path = src[idx+1 : qidx] - if qidx < len(src) { - q, err := url.ParseQuery(src[qidx+1:]) - if err != nil { - return "", true, fmt.Errorf("error parsing GitHub SSH URL: %s", err) - } - - u.RawQuery = q.Encode() - } - - return "git::" + u.String(), true, nil -} diff --git a/config/module/detect_github_test.go b/config/module/detect_github_test.go deleted file mode 100644 index 822e1806d..000000000 --- a/config/module/detect_github_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package module - -import ( - "testing" -) - -func TestGitHubDetector(t *testing.T) { - cases := []struct { - Input string - Output string - }{ - // HTTP - {"github.com/hashicorp/foo", "git::https://github.com/hashicorp/foo.git"}, - {"github.com/hashicorp/foo.git", "git::https://github.com/hashicorp/foo.git"}, - { - "github.com/hashicorp/foo/bar", - "git::https://github.com/hashicorp/foo.git//bar", - }, - { - "github.com/hashicorp/foo?foo=bar", - "git::https://github.com/hashicorp/foo.git?foo=bar", - }, - { - "github.com/hashicorp/foo.git?foo=bar", - "git::https://github.com/hashicorp/foo.git?foo=bar", - }, - - // SSH - {"git@github.com:hashicorp/foo.git", "git::ssh://git@github.com/hashicorp/foo.git"}, - { - "git@github.com:hashicorp/foo.git//bar", - "git::ssh://git@github.com/hashicorp/foo.git//bar", - }, - { - "git@github.com:hashicorp/foo.git?foo=bar", - "git::ssh://git@github.com/hashicorp/foo.git?foo=bar", - }, - } - - pwd := "/pwd" - f := new(GitHubDetector) - for i, tc := range cases { - output, ok, err := f.Detect(tc.Input, pwd) - if err != nil { - t.Fatalf("err: %s", err) - } - if !ok { - t.Fatal("not ok") - } - - if output != tc.Output { - t.Fatalf("%d: bad: %#v", i, output) - } - } -} diff --git a/config/module/detect_test.go b/config/module/detect_test.go deleted file mode 100644 index d2ee8ea1a..000000000 --- a/config/module/detect_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package module - -import ( - "testing" -) - -func TestDetect(t *testing.T) { - cases := []struct { - Input string - Pwd string - Output string - Err bool - }{ - {"./foo", "/foo", "file:///foo/foo", false}, - {"git::./foo", "/foo", "git::file:///foo/foo", false}, - { - "git::github.com/hashicorp/foo", - "", - "git::https://github.com/hashicorp/foo.git", - false, - }, - { - "./foo//bar", - "/foo", - "file:///foo/foo//bar", - false, - }, - { - "git::github.com/hashicorp/foo//bar", - "", - "git::https://github.com/hashicorp/foo.git//bar", - false, - }, - { - "git::https://github.com/hashicorp/consul.git", - "", - "git::https://github.com/hashicorp/consul.git", - false, - }, - } - - for i, tc := range cases { - output, err := Detect(tc.Input, tc.Pwd) - if err != nil != tc.Err { - t.Fatalf("%d: bad err: %s", i, err) - } - if output != tc.Output { - t.Fatalf("%d: bad output: %s\nexpected: %s", i, output, tc.Output) - } - } -} diff --git a/config/module/folder_storage.go b/config/module/folder_storage.go deleted file mode 100644 index 81c9a2ac1..000000000 --- a/config/module/folder_storage.go +++ /dev/null @@ -1,65 +0,0 @@ -package module - -import ( - "crypto/md5" - "encoding/hex" - "fmt" - "os" - "path/filepath" -) - -// FolderStorage is an implementation of the Storage interface that manages -// modules on the disk. -type FolderStorage struct { - // StorageDir is the directory where the modules will be stored. - StorageDir string -} - -// Dir implements Storage.Dir -func (s *FolderStorage) Dir(key string) (d string, e bool, err error) { - d = s.dir(key) - _, err = os.Stat(d) - if err == nil { - // Directory exists - e = true - return - } - if os.IsNotExist(err) { - // Directory doesn't exist - d = "" - e = false - err = nil - return - } - - // An error - d = "" - e = false - return -} - -// Get implements Storage.Get -func (s *FolderStorage) Get(key string, source string, update bool) error { - dir := s.dir(key) - if !update { - if _, err := os.Stat(dir); err == nil { - // If the directory already exists, then we're done since - // we're not updating. - return nil - } else if !os.IsNotExist(err) { - // If the error we got wasn't a file-not-exist error, then - // something went wrong and we should report it. - return fmt.Errorf("Error reading module directory: %s", err) - } - } - - // Get the source. This always forces an update. - return Get(dir, source) -} - -// dir returns the directory name internally that we'll use to map to -// internally. -func (s *FolderStorage) dir(key string) string { - sum := md5.Sum([]byte(key)) - return filepath.Join(s.StorageDir, hex.EncodeToString(sum[:])) -} diff --git a/config/module/folder_storage_test.go b/config/module/folder_storage_test.go deleted file mode 100644 index 7fda6b21a..000000000 --- a/config/module/folder_storage_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package module - -import ( - "os" - "path/filepath" - "testing" -) - -func TestFolderStorage_impl(t *testing.T) { - var _ Storage = new(FolderStorage) -} - -func TestFolderStorage(t *testing.T) { - s := &FolderStorage{StorageDir: tempDir(t)} - - module := testModule("basic") - - // A module shouldn't exist at first... - _, ok, err := s.Dir(module) - if err != nil { - t.Fatalf("err: %s", err) - } - if ok { - t.Fatal("should not exist") - } - - key := "foo" - - // We can get it - err = s.Get(key, module, false) - if err != nil { - t.Fatalf("err: %s", err) - } - - // Now the module exists - dir, ok, err := s.Dir(key) - if err != nil { - t.Fatalf("err: %s", err) - } - if !ok { - t.Fatal("should exist") - } - - mainPath := filepath.Join(dir, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} diff --git a/config/module/get.go b/config/module/get.go index 627d395a9..3820e65f2 100644 --- a/config/module/get.go +++ b/config/module/get.go @@ -1,207 +1,36 @@ package module import ( - "bytes" - "fmt" - "io/ioutil" - "net/url" - "os" - "os/exec" - "path/filepath" - "regexp" - "strings" - "syscall" - - urlhelper "github.com/hashicorp/terraform/helper/url" + "github.com/hashicorp/go-getter" ) -// Getter defines the interface that schemes must implement to download -// and update modules. -type Getter interface { - // Get downloads the given URL into the given directory. This always - // assumes that we're updating and gets the latest version that it can. - // - // The directory may already exist (if we're updating). If it is in a - // format that isn't understood, an error should be returned. Get shouldn't - // simply nuke the directory. - Get(string, *url.URL) error -} - -// Getters is the mapping of scheme to the Getter implementation that will -// be used to get a dependency. -var Getters map[string]Getter - -// forcedRegexp is the regular expression that finds forced getters. This -// syntax is schema::url, example: git::https://foo.com -var forcedRegexp = regexp.MustCompile(`^([A-Za-z]+)::(.+)$`) - -func init() { - httpGetter := new(HttpGetter) - - Getters = map[string]Getter{ - "file": new(FileGetter), - "git": new(GitGetter), - "hg": new(HgGetter), - "http": httpGetter, - "https": httpGetter, - } -} - -// Get downloads the module specified by src into the folder specified by -// dst. If dst already exists, Get will attempt to update it. +// GetMode is an enum that describes how modules are loaded. // -// src is a URL, whereas dst is always just a file path to a folder. This -// folder doesn't need to exist. It will be created if it doesn't exist. -func Get(dst, src string) error { - var force string - force, src = getForcedGetter(src) - - // If there is a subdir component, then we download the root separately - // and then copy over the proper subdir. - var realDst string - src, subDir := getDirSubdir(src) - if subDir != "" { - tmpDir, err := ioutil.TempDir("", "tf") - if err != nil { - return err - } - if err := os.RemoveAll(tmpDir); err != nil { - return err - } - defer os.RemoveAll(tmpDir) - - realDst = dst - dst = tmpDir - } - - u, err := urlhelper.Parse(src) - if err != nil { - return err - } - if force == "" { - force = u.Scheme - } - - g, ok := Getters[force] - if !ok { - return fmt.Errorf( - "module download not supported for scheme '%s'", force) - } - - err = g.Get(dst, u) - if err != nil { - err = fmt.Errorf("error downloading module '%s': %s", src, err) - return err - } - - // If we have a subdir, copy that over - if subDir != "" { - if err := os.RemoveAll(realDst); err != nil { - return err - } - if err := os.MkdirAll(realDst, 0755); err != nil { - return err - } - - return copyDir(realDst, filepath.Join(dst, subDir)) - } - - return nil -} - -// GetCopy is the same as Get except that it downloads a copy of the -// module represented by source. +// GetModeLoad says that modules will not be downloaded or updated, they will +// only be loaded from the storage. // -// This copy will omit and dot-prefixed files (such as .git/, .hg/) and -// can't be updated on its own. -func GetCopy(dst, src string) error { - // Create the temporary directory to do the real Get to - tmpDir, err := ioutil.TempDir("", "tf") - if err != nil { - return err - } - if err := os.RemoveAll(tmpDir); err != nil { - return err - } - defer os.RemoveAll(tmpDir) +// GetModeGet says that modules can be initially downloaded if they don't +// exist, but otherwise to just load from the current version in storage. +// +// GetModeUpdate says that modules should be checked for updates and +// downloaded prior to loading. If there are no updates, we load the version +// from disk, otherwise we download first and then load. +type GetMode byte - // Get to that temporary dir - if err := Get(tmpDir, src); err != nil { - return err - } +const ( + GetModeNone GetMode = iota + GetModeGet + GetModeUpdate +) - // Make sure the destination exists - if err := os.MkdirAll(dst, 0755); err != nil { - return err - } - - // Copy to the final location - return copyDir(dst, tmpDir) -} - -// getRunCommand is a helper that will run a command and capture the output -// in the case an error happens. -func getRunCommand(cmd *exec.Cmd) error { - var buf bytes.Buffer - cmd.Stdout = &buf - cmd.Stderr = &buf - err := cmd.Run() - if err == nil { - return nil - } - if exiterr, ok := err.(*exec.ExitError); ok { - // The program has exited with an exit code != 0 - if status, ok := exiterr.Sys().(syscall.WaitStatus); ok { - return fmt.Errorf( - "%s exited with %d: %s", - cmd.Path, - status.ExitStatus(), - buf.String()) +func getStorage(s getter.Storage, key string, src string, mode GetMode) (string, bool, error) { + // Get the module with the level specified if we were told to. + if mode > GetModeNone { + if err := s.Get(key, src, mode == GetModeUpdate); err != nil { + return "", false, err } } - return fmt.Errorf("error running %s: %s", cmd.Path, buf.String()) -} - -// getDirSubdir takes a source and returns a tuple of the URL without -// the subdir and the URL with the subdir. -func getDirSubdir(src string) (string, string) { - // Calcaulate an offset to avoid accidentally marking the scheme - // as the dir. - var offset int - if idx := strings.Index(src, "://"); idx > -1 { - offset = idx + 3 - } - - // First see if we even have an explicit subdir - idx := strings.Index(src[offset:], "//") - if idx == -1 { - return src, "" - } - - idx += offset - subdir := src[idx+2:] - src = src[:idx] - - // Next, check if we have query parameters and push them onto the - // URL. - if idx = strings.Index(subdir, "?"); idx > -1 { - query := subdir[idx:] - subdir = subdir[:idx] - src += query - } - - return src, subdir -} - -// getForcedGetter takes a source and returns the tuple of the forced -// getter and the raw URL (without the force syntax). -func getForcedGetter(src string) (string, string) { - var forced string - if ms := forcedRegexp.FindStringSubmatch(src); ms != nil { - forced = ms[1] - src = ms[2] - } - - return forced, src + // Get the directory where the module is. + return s.Dir(key) } diff --git a/config/module/get_file.go b/config/module/get_file.go deleted file mode 100644 index 73cb85834..000000000 --- a/config/module/get_file.go +++ /dev/null @@ -1,46 +0,0 @@ -package module - -import ( - "fmt" - "net/url" - "os" - "path/filepath" -) - -// FileGetter is a Getter implementation that will download a module from -// a file scheme. -type FileGetter struct{} - -func (g *FileGetter) Get(dst string, u *url.URL) error { - // The source path must exist and be a directory to be usable. - if fi, err := os.Stat(u.Path); err != nil { - return fmt.Errorf("source path error: %s", err) - } else if !fi.IsDir() { - return fmt.Errorf("source path must be a directory") - } - - fi, err := os.Lstat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - - // If the destination already exists, it must be a symlink - if err == nil { - mode := fi.Mode() - if mode&os.ModeSymlink == 0 { - return fmt.Errorf("destination exists and is not a symlink") - } - - // Remove the destination - if err := os.Remove(dst); err != nil { - return err - } - } - - // Create all the parent directories - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - return err - } - - return os.Symlink(u.Path, dst) -} diff --git a/config/module/get_file_test.go b/config/module/get_file_test.go deleted file mode 100644 index 4c9f6126a..000000000 --- a/config/module/get_file_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package module - -import ( - "os" - "path/filepath" - "testing" -) - -func TestFileGetter_impl(t *testing.T) { - var _ Getter = new(FileGetter) -} - -func TestFileGetter(t *testing.T) { - g := new(FileGetter) - dst := tempDir(t) - - // With a dir that doesn't exist - if err := g.Get(dst, testModuleURL("basic")); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the destination folder is a symlink - fi, err := os.Lstat(dst) - if err != nil { - t.Fatalf("err: %s", err) - } - if fi.Mode()&os.ModeSymlink == 0 { - t.Fatal("destination is not a symlink") - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestFileGetter_sourceFile(t *testing.T) { - g := new(FileGetter) - dst := tempDir(t) - - // With a source URL that is a path to a file - u := testModuleURL("basic") - u.Path += "/main.tf" - if err := g.Get(dst, u); err == nil { - t.Fatal("should error") - } -} - -func TestFileGetter_sourceNoExist(t *testing.T) { - g := new(FileGetter) - dst := tempDir(t) - - // With a source URL that doesn't exist - u := testModuleURL("basic") - u.Path += "/main" - if err := g.Get(dst, u); err == nil { - t.Fatal("should error") - } -} - -func TestFileGetter_dir(t *testing.T) { - g := new(FileGetter) - dst := tempDir(t) - - if err := os.MkdirAll(dst, 0755); err != nil { - t.Fatalf("err: %s", err) - } - - // With a dir that exists that isn't a symlink - if err := g.Get(dst, testModuleURL("basic")); err == nil { - t.Fatal("should error") - } -} - -func TestFileGetter_dirSymlink(t *testing.T) { - g := new(FileGetter) - dst := tempDir(t) - dst2 := tempDir(t) - - // Make parents - if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { - t.Fatalf("err: %s", err) - } - if err := os.MkdirAll(dst2, 0755); err != nil { - t.Fatalf("err: %s", err) - } - - // Make a symlink - if err := os.Symlink(dst2, dst); err != nil { - t.Fatalf("err: %s", err) - } - - // With a dir that exists that isn't a symlink - if err := g.Get(dst, testModuleURL("basic")); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} diff --git a/config/module/get_git.go b/config/module/get_git.go deleted file mode 100644 index 5ab27ba0b..000000000 --- a/config/module/get_git.go +++ /dev/null @@ -1,74 +0,0 @@ -package module - -import ( - "fmt" - "net/url" - "os" - "os/exec" -) - -// GitGetter is a Getter implementation that will download a module from -// a git repository. -type GitGetter struct{} - -func (g *GitGetter) Get(dst string, u *url.URL) error { - if _, err := exec.LookPath("git"); err != nil { - return fmt.Errorf("git must be available and on the PATH") - } - - // Extract some query parameters we use - var ref string - q := u.Query() - if len(q) > 0 { - ref = q.Get("ref") - q.Del("ref") - - // Copy the URL - var newU url.URL = *u - u = &newU - u.RawQuery = q.Encode() - } - - // First: clone or update the repository - _, err := os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - err = g.update(dst, u) - } else { - err = g.clone(dst, u) - } - if err != nil { - return err - } - - // Next: check out the proper tag/branch if it is specified, and checkout - if ref == "" { - return nil - } - - return g.checkout(dst, ref) -} - -func (g *GitGetter) checkout(dst string, ref string) error { - cmd := exec.Command("git", "checkout", ref) - cmd.Dir = dst - return getRunCommand(cmd) -} - -func (g *GitGetter) clone(dst string, u *url.URL) error { - cmd := exec.Command("git", "clone", u.String(), dst) - return getRunCommand(cmd) -} - -func (g *GitGetter) update(dst string, u *url.URL) error { - // We have to be on a branch to pull - if err := g.checkout(dst, "master"); err != nil { - return err - } - - cmd := exec.Command("git", "pull", "--ff-only") - cmd.Dir = dst - return getRunCommand(cmd) -} diff --git a/config/module/get_git_test.go b/config/module/get_git_test.go deleted file mode 100644 index 3885ff8e7..000000000 --- a/config/module/get_git_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package module - -import ( - "os" - "os/exec" - "path/filepath" - "testing" -) - -var testHasGit bool - -func init() { - if _, err := exec.LookPath("git"); err == nil { - testHasGit = true - } -} - -func TestGitGetter_impl(t *testing.T) { - var _ Getter = new(GitGetter) -} - -func TestGitGetter(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(GitGetter) - dst := tempDir(t) - - // Git doesn't allow nested ".git" directories so we do some hackiness - // here to get around that... - moduleDir := filepath.Join(fixtureDir, "basic-git") - oldName := filepath.Join(moduleDir, "DOTgit") - newName := filepath.Join(moduleDir, ".git") - if err := os.Rename(oldName, newName); err != nil { - t.Fatalf("err: %s", err) - } - defer os.Rename(newName, oldName) - - // With a dir that doesn't exist - if err := g.Get(dst, testModuleURL("basic-git")); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_branch(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(GitGetter) - dst := tempDir(t) - - // Git doesn't allow nested ".git" directories so we do some hackiness - // here to get around that... - moduleDir := filepath.Join(fixtureDir, "basic-git") - oldName := filepath.Join(moduleDir, "DOTgit") - newName := filepath.Join(moduleDir, ".git") - if err := os.Rename(oldName, newName); err != nil { - t.Fatalf("err: %s", err) - } - defer os.Rename(newName, oldName) - - url := testModuleURL("basic-git") - q := url.Query() - q.Add("ref", "test-branch") - url.RawQuery = q.Encode() - - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main_branch.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "main_branch.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGitGetter_tag(t *testing.T) { - if !testHasGit { - t.Log("git not found, skipping") - t.Skip() - } - - g := new(GitGetter) - dst := tempDir(t) - - // Git doesn't allow nested ".git" directories so we do some hackiness - // here to get around that... - moduleDir := filepath.Join(fixtureDir, "basic-git") - oldName := filepath.Join(moduleDir, "DOTgit") - newName := filepath.Join(moduleDir, ".git") - if err := os.Rename(oldName, newName); err != nil { - t.Fatalf("err: %s", err) - } - defer os.Rename(newName, oldName) - - url := testModuleURL("basic-git") - q := url.Query() - q.Add("ref", "v1.0") - url.RawQuery = q.Encode() - - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main_tag1.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "main_tag1.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} diff --git a/config/module/get_hg.go b/config/module/get_hg.go deleted file mode 100644 index f74c14093..000000000 --- a/config/module/get_hg.go +++ /dev/null @@ -1,89 +0,0 @@ -package module - -import ( - "fmt" - "net/url" - "os" - "os/exec" - "runtime" - - urlhelper "github.com/hashicorp/terraform/helper/url" -) - -// HgGetter is a Getter implementation that will download a module from -// a Mercurial repository. -type HgGetter struct{} - -func (g *HgGetter) Get(dst string, u *url.URL) error { - if _, err := exec.LookPath("hg"); err != nil { - return fmt.Errorf("hg must be available and on the PATH") - } - - newURL, err := urlhelper.Parse(u.String()) - if err != nil { - return err - } - if fixWindowsDrivePath(newURL) { - // See valid file path form on http://www.selenic.com/hg/help/urls - newURL.Path = fmt.Sprintf("/%s", newURL.Path) - } - - // Extract some query parameters we use - var rev string - q := newURL.Query() - if len(q) > 0 { - rev = q.Get("rev") - q.Del("rev") - - newURL.RawQuery = q.Encode() - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - if err != nil { - if err := g.clone(dst, newURL); err != nil { - return err - } - } - - if err := g.pull(dst, newURL); err != nil { - return err - } - - return g.update(dst, newURL, rev) -} - -func (g *HgGetter) clone(dst string, u *url.URL) error { - cmd := exec.Command("hg", "clone", "-U", u.String(), dst) - return getRunCommand(cmd) -} - -func (g *HgGetter) pull(dst string, u *url.URL) error { - cmd := exec.Command("hg", "pull") - cmd.Dir = dst - return getRunCommand(cmd) -} - -func (g *HgGetter) update(dst string, u *url.URL, rev string) error { - args := []string{"update"} - if rev != "" { - args = append(args, rev) - } - - cmd := exec.Command("hg", args...) - cmd.Dir = dst - return getRunCommand(cmd) -} - -func fixWindowsDrivePath(u *url.URL) bool { - // hg assumes a file:/// prefix for Windows drive letter file paths. - // (e.g. file:///c:/foo/bar) - // If the URL Path does not begin with a '/' character, the resulting URL - // path will have a file:// prefix. (e.g. file://c:/foo/bar) - // See http://www.selenic.com/hg/help/urls and the examples listed in - // http://selenic.com/repo/hg-stable/file/1265a3a71d75/mercurial/util.py#l1936 - return runtime.GOOS == "windows" && u.Scheme == "file" && - len(u.Path) > 1 && u.Path[0] != '/' && u.Path[1] == ':' -} diff --git a/config/module/get_hg_test.go b/config/module/get_hg_test.go deleted file mode 100644 index d7125bde2..000000000 --- a/config/module/get_hg_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package module - -import ( - "os" - "os/exec" - "path/filepath" - "testing" -) - -var testHasHg bool - -func init() { - if _, err := exec.LookPath("hg"); err == nil { - testHasHg = true - } -} - -func TestHgGetter_impl(t *testing.T) { - var _ Getter = new(HgGetter) -} - -func TestHgGetter(t *testing.T) { - t.Parallel() - - if !testHasHg { - t.Log("hg not found, skipping") - t.Skip() - } - - g := new(HgGetter) - dst := tempDir(t) - - // With a dir that doesn't exist - if err := g.Get(dst, testModuleURL("basic-hg")); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestHgGetter_branch(t *testing.T) { - t.Parallel() - - if !testHasHg { - t.Log("hg not found, skipping") - t.Skip() - } - - g := new(HgGetter) - dst := tempDir(t) - - url := testModuleURL("basic-hg") - q := url.Query() - q.Add("rev", "test-branch") - url.RawQuery = q.Encode() - - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main_branch.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - // Get again should work - if err := g.Get(dst, url); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath = filepath.Join(dst, "main_branch.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} diff --git a/config/module/get_http.go b/config/module/get_http.go deleted file mode 100644 index be65d921a..000000000 --- a/config/module/get_http.go +++ /dev/null @@ -1,173 +0,0 @@ -package module - -import ( - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" -) - -// HttpGetter is a Getter implementation that will download a module from -// an HTTP endpoint. The protocol for downloading a module from an HTTP -// endpoing is as follows: -// -// An HTTP GET request is made to the URL with the additional GET parameter -// "terraform-get=1". This lets you handle that scenario specially if you -// wish. The response must be a 2xx. -// -// First, a header is looked for "X-Terraform-Get" which should contain -// a source URL to download. -// -// If the header is not present, then a meta tag is searched for named -// "terraform-get" and the content should be a source URL. -// -// The source URL, whether from the header or meta tag, must be a fully -// formed URL. The shorthand syntax of "github.com/foo/bar" or relative -// paths are not allowed. -type HttpGetter struct{} - -func (g *HttpGetter) Get(dst string, u *url.URL) error { - // Copy the URL so we can modify it - var newU url.URL = *u - u = &newU - - // Add terraform-get to the parameter. - q := u.Query() - q.Add("terraform-get", "1") - u.RawQuery = q.Encode() - - // Get the URL - resp, err := http.Get(u.String()) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("bad response code: %d", resp.StatusCode) - } - - // Extract the source URL - var source string - if v := resp.Header.Get("X-Terraform-Get"); v != "" { - source = v - } else { - source, err = g.parseMeta(resp.Body) - if err != nil { - return err - } - } - if source == "" { - return fmt.Errorf("no source URL was returned") - } - - // If there is a subdir component, then we download the root separately - // into a temporary directory, then copy over the proper subdir. - source, subDir := getDirSubdir(source) - if subDir == "" { - return Get(dst, source) - } - - // We have a subdir, time to jump some hoops - return g.getSubdir(dst, source, subDir) -} - -// getSubdir downloads the source into the destination, but with -// the proper subdir. -func (g *HttpGetter) getSubdir(dst, source, subDir string) error { - // Create a temporary directory to store the full source - td, err := ioutil.TempDir("", "tf") - if err != nil { - return err - } - defer os.RemoveAll(td) - - // Download that into the given directory - if err := Get(td, source); err != nil { - return err - } - - // Make sure the subdir path actually exists - sourcePath := filepath.Join(td, subDir) - if _, err := os.Stat(sourcePath); err != nil { - return fmt.Errorf( - "Error downloading %s: %s", source, err) - } - - // Copy the subdirectory into our actual destination. - if err := os.RemoveAll(dst); err != nil { - return err - } - - // Make the final destination - if err := os.MkdirAll(dst, 0755); err != nil { - return err - } - - return copyDir(dst, sourcePath) -} - -// parseMeta looks for the first meta tag in the given reader that -// will give us the source URL. -func (g *HttpGetter) parseMeta(r io.Reader) (string, error) { - d := xml.NewDecoder(r) - d.CharsetReader = charsetReader - d.Strict = false - var err error - var t xml.Token - for { - t, err = d.Token() - if err != nil { - if err == io.EOF { - err = nil - } - return "", err - } - if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { - return "", nil - } - if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { - return "", nil - } - e, ok := t.(xml.StartElement) - if !ok || !strings.EqualFold(e.Name.Local, "meta") { - continue - } - if attrValue(e.Attr, "name") != "terraform-get" { - continue - } - if f := attrValue(e.Attr, "content"); f != "" { - return f, nil - } - } -} - -// attrValue returns the attribute value for the case-insensitive key -// `name', or the empty string if nothing is found. -func attrValue(attrs []xml.Attr, name string) string { - for _, a := range attrs { - if strings.EqualFold(a.Name.Local, name) { - return a.Value - } - } - return "" -} - -// charsetReader returns a reader for the given charset. Currently -// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful -// error which is printed by go get, so the user can find why the package -// wasn't downloaded if the encoding is not supported. Note that, in -// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters -// greater than 0x7f are not rejected). -func charsetReader(charset string, input io.Reader) (io.Reader, error) { - switch strings.ToLower(charset) { - case "ascii": - return input, nil - default: - return nil, fmt.Errorf("can't decode XML document using charset %q", charset) - } -} diff --git a/config/module/get_http_test.go b/config/module/get_http_test.go deleted file mode 100644 index 5f2590f48..000000000 --- a/config/module/get_http_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package module - -import ( - "fmt" - "net" - "net/http" - "net/url" - "os" - "path/filepath" - "testing" -) - -func TestHttpGetter_impl(t *testing.T) { - var _ Getter = new(HttpGetter) -} - -func TestHttpGetter_header(t *testing.T) { - ln := testHttpServer(t) - defer ln.Close() - - g := new(HttpGetter) - dst := tempDir(t) - - var u url.URL - u.Scheme = "http" - u.Host = ln.Addr().String() - u.Path = "/header" - - // Get it! - if err := g.Get(dst, &u); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestHttpGetter_meta(t *testing.T) { - ln := testHttpServer(t) - defer ln.Close() - - g := new(HttpGetter) - dst := tempDir(t) - - var u url.URL - u.Scheme = "http" - u.Host = ln.Addr().String() - u.Path = "/meta" - - // Get it! - if err := g.Get(dst, &u); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestHttpGetter_metaSubdir(t *testing.T) { - ln := testHttpServer(t) - defer ln.Close() - - g := new(HttpGetter) - dst := tempDir(t) - - var u url.URL - u.Scheme = "http" - u.Host = ln.Addr().String() - u.Path = "/meta-subdir" - - // Get it! - if err := g.Get(dst, &u); err != nil { - t.Fatalf("err: %s", err) - } - - // Verify the main file exists - mainPath := filepath.Join(dst, "sub.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestHttpGetter_none(t *testing.T) { - ln := testHttpServer(t) - defer ln.Close() - - g := new(HttpGetter) - dst := tempDir(t) - - var u url.URL - u.Scheme = "http" - u.Host = ln.Addr().String() - u.Path = "/none" - - // Get it! - if err := g.Get(dst, &u); err == nil { - t.Fatal("should error") - } -} - -func testHttpServer(t *testing.T) net.Listener { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("err: %s", err) - } - - mux := http.NewServeMux() - mux.HandleFunc("/header", testHttpHandlerHeader) - mux.HandleFunc("/meta", testHttpHandlerMeta) - mux.HandleFunc("/meta-subdir", testHttpHandlerMetaSubdir) - - var server http.Server - server.Handler = mux - go server.Serve(ln) - - return ln -} - -func testHttpHandlerHeader(w http.ResponseWriter, r *http.Request) { - w.Header().Add("X-Terraform-Get", testModuleURL("basic").String()) - w.WriteHeader(200) -} - -func testHttpHandlerMeta(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic").String()))) -} - -func testHttpHandlerMetaSubdir(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(fmt.Sprintf(testHttpMetaStr, testModuleURL("basic//subdir").String()))) -} - -func testHttpHandlerNone(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(testHttpNoneStr)) -} - -const testHttpMetaStr = ` - - - - - -` - -const testHttpNoneStr = ` - - - - -` diff --git a/config/module/get_test.go b/config/module/get_test.go deleted file mode 100644 index b403c835c..000000000 --- a/config/module/get_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package module - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestGet_badSchema(t *testing.T) { - dst := tempDir(t) - u := testModule("basic") - u = strings.Replace(u, "file", "nope", -1) - - if err := Get(dst, u); err == nil { - t.Fatal("should error") - } -} - -func TestGet_file(t *testing.T) { - dst := tempDir(t) - u := testModule("basic") - - if err := Get(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGet_fileForced(t *testing.T) { - dst := tempDir(t) - u := testModule("basic") - u = "file::" + u - - if err := Get(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGet_fileSubdir(t *testing.T) { - dst := tempDir(t) - u := testModule("basic//subdir") - - if err := Get(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath := filepath.Join(dst, "sub.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGetCopy_dot(t *testing.T) { - dst := tempDir(t) - u := testModule("basic-dot") - - if err := GetCopy(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath = filepath.Join(dst, "foo.tf") - if _, err := os.Stat(mainPath); err == nil { - t.Fatal("should not have foo.tf") - } -} - -func TestGetCopy_file(t *testing.T) { - dst := tempDir(t) - u := testModule("basic") - - if err := GetCopy(dst, u); err != nil { - t.Fatalf("err: %s", err) - } - - mainPath := filepath.Join(dst, "main.tf") - if _, err := os.Stat(mainPath); err != nil { - t.Fatalf("err: %s", err) - } -} - -func TestGetDirSubdir(t *testing.T) { - cases := []struct { - Input string - Dir, Sub string - }{ - { - "hashicorp.com", - "hashicorp.com", "", - }, - { - "hashicorp.com//foo", - "hashicorp.com", "foo", - }, - { - "hashicorp.com//foo?bar=baz", - "hashicorp.com?bar=baz", "foo", - }, - { - "file://foo//bar", - "file://foo", "bar", - }, - } - - for i, tc := range cases { - adir, asub := getDirSubdir(tc.Input) - if adir != tc.Dir { - t.Fatalf("%d: bad dir: %#v", i, adir) - } - if asub != tc.Sub { - t.Fatalf("%d: bad sub: %#v", i, asub) - } - } -} diff --git a/config/module/module_test.go b/config/module/module_test.go index f1517e480..89fee6ec5 100644 --- a/config/module/module_test.go +++ b/config/module/module_test.go @@ -2,13 +2,12 @@ package module import ( "io/ioutil" - "net/url" "os" "path/filepath" "testing" + "github.com/hashicorp/go-getter" "github.com/hashicorp/terraform/config" - urlhelper "github.com/hashicorp/terraform/helper/url" ) const fixtureDir = "./test-fixtures" @@ -34,24 +33,6 @@ func testConfig(t *testing.T, n string) *config.Config { return c } -func testModule(n string) string { - p := filepath.Join(fixtureDir, n) - p, err := filepath.Abs(p) - if err != nil { - panic(err) - } - return fmtFileURL(p) -} - -func testModuleURL(n string) *url.URL { - u, err := urlhelper.Parse(testModule(n)) - if err != nil { - panic(err) - } - - return u -} - -func testStorage(t *testing.T) Storage { - return &FolderStorage{StorageDir: tempDir(t)} +func testStorage(t *testing.T) getter.Storage { + return &getter.FolderStorage{StorageDir: tempDir(t)} } diff --git a/config/module/storage.go b/config/module/storage.go deleted file mode 100644 index 9c752f630..000000000 --- a/config/module/storage.go +++ /dev/null @@ -1,25 +0,0 @@ -package module - -// Storage is an interface that knows how to lookup downloaded modules -// as well as download and update modules from their sources into the -// proper location. -type Storage interface { - // Dir returns the directory on local disk where the modulue source - // can be loaded from. - Dir(string) (string, bool, error) - - // Get will download and optionally update the given module. - Get(string, string, bool) error -} - -func getStorage(s Storage, key string, src string, mode GetMode) (string, bool, error) { - // Get the module with the level specified if we were told to. - if mode > GetModeNone { - if err := s.Get(key, src, mode == GetModeUpdate); err != nil { - return "", false, err - } - } - - // Get the directory where the module is. - return s.Dir(key) -} diff --git a/config/module/tree.go b/config/module/tree.go index d7b3ac966..6a75c19c2 100644 --- a/config/module/tree.go +++ b/config/module/tree.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/hashicorp/go-getter" "github.com/hashicorp/terraform/config" ) @@ -27,25 +28,6 @@ type Tree struct { lock sync.RWMutex } -// GetMode is an enum that describes how modules are loaded. -// -// GetModeLoad says that modules will not be downloaded or updated, they will -// only be loaded from the storage. -// -// GetModeGet says that modules can be initially downloaded if they don't -// exist, but otherwise to just load from the current version in storage. -// -// GetModeUpdate says that modules should be checked for updates and -// downloaded prior to loading. If there are no updates, we load the version -// from disk, otherwise we download first and then load. -type GetMode byte - -const ( - GetModeNone GetMode = iota - GetModeGet - GetModeUpdate -) - // NewTree returns a new Tree for the given config structure. func NewTree(name string, c *config.Config) *Tree { return &Tree{config: c, name: name} @@ -136,7 +118,7 @@ func (t *Tree) Name() string { // module trees inherently require the configuration to be in a reasonably // sane state: no circular dependencies, proper module sources, etc. A full // suite of validations can be done by running Validate (after loading). -func (t *Tree) Load(s Storage, mode GetMode) error { +func (t *Tree) Load(s getter.Storage, mode GetMode) error { t.lock.Lock() defer t.lock.Unlock() @@ -159,15 +141,15 @@ func (t *Tree) Load(s Storage, mode GetMode) error { path = append(path, m.Name) // Split out the subdir if we have one - source, subDir := getDirSubdir(m.Source) + source, subDir := getter.SourceDirSubdir(m.Source) - source, err := Detect(source, t.config.Dir) + source, err := getter.Detect(source, t.config.Dir, getter.Detectors) if err != nil { return fmt.Errorf("module %s: %s", m.Name, err) } // Check if the detector introduced something new. - source, subDir2 := getDirSubdir(source) + source, subDir2 := getter.SourceDirSubdir(source) if subDir2 != "" { subDir = filepath.Join(subDir2, subDir) }