Merge pull request #11388 from hashicorp/f-go-getter

dependency: update go-getter

vendor/github.com/hashicorp/go-getter/README.md

@@ -210,6 +210,12 @@ None
     a commit SHA, a branch name, etc. If it is a named ref such as a branch
     name, go-getter will update it to the latest on each get.
 
+  * `sshkey` - An SSH private key to use during clones. The provided key must
+    be a base64-encoded string. For example, to generate a suitable `sshkey`
+    from a private key file on disk, you would run `base64 -w0 <file>`.
+
+    **Note**: Git 2.3+ is required to use this feature.
+
 ### Mercurial (`hg`)
 
   * `rev` - The Mercurial revision to checkout.

vendor/github.com/hashicorp/go-getter/client.go

@@ -222,13 +222,18 @@ func (c *Client) Get() error {
 		checksumValue = b
 	}
 
-	// For now, any means file. In the future, we'll ask the getter
-	// what it thinks it is.
 	if mode == ClientModeAny {
-		mode = ClientModeFile
+		// Ask the getter which client mode to use
+		mode, err = g.ClientMode(u)
+		if err != nil {
+			return err
+		}
 
-		// Destination is the base name of the URL path
-		dst = filepath.Join(dst, filepath.Base(u.Path))
+		// Destination is the base name of the URL path in "any" mode when
+		// a file source is detected.
+		if mode == ClientModeFile {
+			dst = filepath.Join(dst, filepath.Base(u.Path))
+		}
 	}
 
 	// If we're not downloading a directory, then just download the file

vendor/github.com/hashicorp/go-getter/get.go

@@ -35,6 +35,10 @@ type Getter interface {
 	// reference a single file. If possible, the Getter should check if
 	// the remote end contains the same file and no-op this operation.
 	GetFile(string, *url.URL) error
+
+	// ClientMode returns the mode based on the given URL. This is used to
+	// allow clients to let the getters decide which mode to use.
+	ClientMode(*url.URL) (ClientMode, error)
 }
 
 // Getters is the mapping of scheme to the Getter implementation that will

vendor/github.com/hashicorp/go-getter/get_file.go

@@ -1,8 +1,32 @@
 package getter
 
+import (
+	"net/url"
+	"os"
+)
+
 // FileGetter is a Getter implementation that will download a module from
 // a file scheme.
 type FileGetter struct {
 	// Copy, if set to true, will copy data instead of using a symlink
 	Copy bool
 }
+
+func (g *FileGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+
+	// Check if the source is a directory.
+	if fi.IsDir() {
+		return ClientModeDir, nil
+	}
+
+	return ClientModeFile, nil
+}

vendor/github.com/hashicorp/go-getter/get_file_unix.go

@@ -55,7 +55,7 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 		path = u.RawPath
 	}
 
-	// The source path must exist and be a directory to be usable.
+	// The source path must exist and be a file to be usable.
 	if fi, err := os.Stat(path); err != nil {
 		return fmt.Errorf("source path error: %s", err)
 	} else if fi.IsDir() {

vendor/github.com/hashicorp/go-getter/get_git.go

@@ -1,58 +1,105 @@
 package getter
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
+	"github.com/hashicorp/go-version"
 )
 
 // GitGetter is a Getter implementation that will download a module from
 // a git repository.
 type GitGetter struct{}
 
+func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
 func (g *GitGetter) Get(dst string, u *url.URL) error {
 	if _, err := exec.LookPath("git"); err != nil {
 		return fmt.Errorf("git must be available and on the PATH")
 	}
 
 	// Extract some query parameters we use
-	var ref string
+	var ref, sshKey string
 	q := u.Query()
 	if len(q) > 0 {
 		ref = q.Get("ref")
 		q.Del("ref")
 
+		sshKey = q.Get("sshkey")
+		q.Del("sshkey")
+
 		// Copy the URL
 		var newU url.URL = *u
 		u = &newU
 		u.RawQuery = q.Encode()
 	}
 
-	// First: clone or update the repository
+	var sshKeyFile string
+	if sshKey != "" {
+		// Check that the git version is sufficiently new.
+		if err := checkGitVersion("2.3"); err != nil {
+			return fmt.Errorf("Error using ssh key: %v", err)
+		}
+
+		// We have an SSH key - decode it.
+		raw, err := base64.StdEncoding.DecodeString(sshKey)
+		if err != nil {
+			return err
+		}
+
+		// Create a temp file for the key and ensure it is removed.
+		fh, err := ioutil.TempFile("", "go-getter")
+		if err != nil {
+			return err
+		}
+		sshKeyFile = fh.Name()
+		defer os.Remove(sshKeyFile)
+
+		// Set the permissions prior to writing the key material.
+		if err := os.Chmod(sshKeyFile, 0600); err != nil {
+			return err
+		}
+
+		// Write the raw key into the temp file.
+		_, err = fh.Write(raw)
+		fh.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Clone or update the repository
 	_, err := os.Stat(dst)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	if err == nil {
-		err = g.update(dst, ref)
+		err = g.update(dst, sshKeyFile, ref)
 	} else {
-		err = g.clone(dst, u)
+		err = g.clone(dst, sshKeyFile, u)
 	}
 	if err != nil {
 		return err
 	}
 
 	// Next: check out the proper tag/branch if it is specified, and checkout
-	if ref == "" {
-		return nil
+	if ref != "" {
+		if err := g.checkout(dst, ref); err != nil {
+			return err
+		}
 	}
 
-	return g.checkout(dst, ref)
+	// Lastly, download any/all submodules.
+	return g.fetchSubmodules(dst, sshKeyFile)
 }
 
 // GetFile for Git doesn't support updating at this time. It will download
@@ -92,16 +139,18 @@ func (g *GitGetter) checkout(dst string, ref string) error {
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) clone(dst string, u *url.URL) error {
+func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error {
 	cmd := exec.Command("git", "clone", u.String(), dst)
+	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) update(dst string, ref string) error {
+func (g *GitGetter) update(dst, sshKeyFile, ref string) error {
 	// Determine if we're a branch. If we're NOT a branch, then we just
 	// switch to master prior to checking out
 	cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
 	cmd.Dir = dst
+
 	if getRunCommand(cmd) != nil {
 		// Not a branch, switch to master. This will also catch non-existent
 		// branches, in which case we want to switch to master and then
@@ -116,5 +165,61 @@ func (g *GitGetter) update(dst string, ref string) error {
 
 	cmd = exec.Command("git", "pull", "--ff-only")
 	cmd.Dir = dst
+	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
+
+// fetchSubmodules downloads any configured submodules recursively.
+func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error {
+	cmd := exec.Command("git", "submodule", "update", "--init", "--recursive")
+	cmd.Dir = dst
+	setupGitEnv(cmd, sshKeyFile)
+	return getRunCommand(cmd)
+}
+
+// setupGitEnv sets up the environment for the given command. This is used to
+// pass configuration data to git and ssh and enables advanced cloning methods.
+func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
+	var sshOpts []string
+
+	if sshKeyFile != "" {
+		// We have an SSH key temp file configured, tell ssh about this.
+		sshOpts = append(sshOpts, "-i", sshKeyFile)
+	}
+
+	cmd.Env = append(os.Environ(),
+		// Set the ssh command to use for clones.
+		"GIT_SSH_COMMAND=ssh "+strings.Join(sshOpts, " "),
+	)
+}
+
+// checkGitVersion is used to check the version of git installed on the system
+// against a known minimum version. Returns an error if the installed version
+// is older than the given minimum.
+func checkGitVersion(min string) error {
+	want, err := version.NewVersion(min)
+	if err != nil {
+		return err
+	}
+
+	out, err := exec.Command("git", "version").Output()
+	if err != nil {
+		return err
+	}
+
+	fields := strings.Fields(string(out))
+	if len(fields) != 3 {
+		return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
+	}
+
+	have, err := version.NewVersion(fields[2])
+	if err != nil {
+		return err
+	}
+
+	if have.LessThan(want) {
+		return fmt.Errorf("Required git version = %s, have %s", want, have)
+	}
+
+	return nil
+}

vendor/github.com/hashicorp/go-getter/get_hg.go

@@ -16,6 +16,10 @@ import (
 // a Mercurial repository.
 type HgGetter struct{}
 
+func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
 func (g *HgGetter) Get(dst string, u *url.URL) error {
 	if _, err := exec.LookPath("hg"); err != nil {
 		return fmt.Errorf("hg must be available and on the PATH")

vendor/github.com/hashicorp/go-getter/get_http.go

@@ -38,6 +38,13 @@ type HttpGetter struct {
 	Netrc bool
 }
 
+func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	if strings.HasSuffix(u.Path, "/") {
+		return ClientModeDir, nil
+	}
+	return ClientModeFile, nil
+}
+
 func (g *HttpGetter) Get(dst string, u *url.URL) error {
 	// Copy the URL so we can modify it
 	var newU url.URL = *u

vendor/github.com/hashicorp/go-getter/get_mock.go

@@ -43,3 +43,10 @@ func (g *MockGetter) GetFile(dst string, u *url.URL) error {
 	}
 	return g.GetFileErr
 }
+
+func (g *MockGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	if l := len(u.Path); l > 0 && u.Path[l-1:] == "/" {
+		return ClientModeDir, nil
+	}
+	return ClientModeFile, nil
+}

vendor/github.com/hashicorp/go-getter/get_s3.go

@@ -20,6 +20,45 @@ import (
 // a S3 bucket.
 type S3Getter struct{}
 
+func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
+	// Parse URL
+	region, bucket, path, _, creds, err := g.parseUrl(u)
+	if err != nil {
+		return 0, err
+	}
+
+	// Create client config
+	config := g.getAWSConfig(region, creds)
+	sess := session.New(config)
+	client := s3.New(sess)
+
+	// List the object(s) at the given prefix
+	req := &s3.ListObjectsInput{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(path),
+	}
+	resp, err := client.ListObjects(req)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, o := range resp.Contents {
+		// Use file mode on exact match.
+		if *o.Key == path {
+			return ClientModeFile, nil
+		}
+
+		// Use dir mode if child keys are found.
+		if strings.HasPrefix(*o.Key, path+"/") {
+			return ClientModeDir, nil
+		}
+	}
+
+	// There was no match, so just return file mode. The download is going
+	// to fail but we will let S3 return the proper error later.
+	return ClientModeFile, nil
+}
+
 func (g *S3Getter) Get(dst string, u *url.URL) error {
 	// Parse URL
 	region, bucket, path, _, creds, err := g.parseUrl(u)

vendor/vendor.json

@@ -1619,10 +1619,10 @@
 			"revision": "875fb671b3ddc66f8e2f0acc33829c8cb989a38d"
 		},
 		{
-			"checksumSHA1": "aOWXSbYAdK3PBSMNFiK2ze4lPEc=",
+			"checksumSHA1": "N+MSF+0nq3XFXaUCOooF9acE+ZI=",
 			"path": "github.com/hashicorp/go-getter",
-			"revision": "2fbd997432e72fe36060c8f07ec1eaf98d098177",
-			"revisionTime": "2016-09-12T21:42:52Z"
+			"revision": "cc80f38c726badeae53775d179755e1c4953d6cf",
+			"revisionTime": "2017-01-26T00:13:12Z"
 		},
 		{
 			"checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=",