getmodules: Inline our own fork of getter.GitGetter

This is a pragmatic temporary solution to allow us to more quickly resolve
an upstream regression in go-getter locally within Terraform, so that the
work to upstream it for other callers can happen asynchronously and with
less time pressure.

This commit doesn't yet include any changes to address the bug, and
instead aims to be functionally equivalent to getter.GitGetter. A
subsequent commit will then address the regression, so that the diff of
that commit will be easier to apply later to the upstream to get the same
effect there.
This commit is contained in:
Martin Atkins 2021-12-13 15:31:30 -08:00
parent 8b8fe2771f
commit b0ff17ef2a
4 changed files with 1108 additions and 2 deletions

2
go.mod
View File

@ -41,6 +41,7 @@ require (
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/go-plugin v1.4.3
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/hashicorp/go-safetemp v1.0.0
github.com/hashicorp/go-tfe v0.21.0
github.com/hashicorp/go-uuid v1.0.2
github.com/hashicorp/go-version v1.3.0
@ -144,7 +145,6 @@ require (
github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
github.com/hashicorp/go-msgpack v0.5.4 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-slug v0.7.0 // indirect
github.com/hashicorp/golang-lru v0.5.1 // indirect
github.com/hashicorp/jsonapi v0.0.0-20210826224640-ee7dae0fb22d // indirect

View File

@ -73,7 +73,7 @@ var goGetterDecompressors = map[string]getter.Decompressor{
var goGetterGetters = map[string]getter.Getter{
"file": new(getter.FileGetter),
"gcs": new(getter.GCSGetter),
"git": new(getter.GitGetter),
"git": new(gitGetter),
"hg": new(getter.HgGetter),
"s3": new(getter.S3Getter),
"http": getterHTTPGetter,

View File

@ -0,0 +1,382 @@
package getmodules
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io/ioutil"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strconv"
"strings"
"syscall"
getter "github.com/hashicorp/go-getter"
urlhelper "github.com/hashicorp/go-getter/helper/url"
safetemp "github.com/hashicorp/go-safetemp"
version "github.com/hashicorp/go-version"
)
// getter is our base getter; it regroups
// fields all getters have in common.
type getterCommon struct {
client *getter.Client
}
func (g *getterCommon) SetClient(c *getter.Client) { g.client = c }
// Context tries to returns the Contex from the getter's
// client. otherwise context.Background() is returned.
func (g *getterCommon) Context() context.Context {
if g == nil || g.client == nil {
return context.Background()
}
return g.client.Ctx
}
// gitGetter is a temporary fork of getter.GitGetter to allow us to tactically
// fix https://github.com/hashicorp/terraform/issues/30119 only within
// Terraform.
//
// This should be only a brief workaround to help us decouple work on the
// Terraform CLI v1.1.1 release so that we can get it done without having to
// coordinate with every other go-getter caller first. However, this fork
// should be healed promptly after v1.1.1 by upstreaming something like this
// fix into upstream go-getter, so that other go-getter callers can also
// benefit from it.
type gitGetter struct {
getterCommon
}
var defaultBranchRegexp = regexp.MustCompile(`\s->\sorigin/(.*)`)
var lsRemoteSymRefRegexp = regexp.MustCompile(`ref: refs/heads/([^\s]+).*`)
func (g *gitGetter) ClientMode(_ *url.URL) (getter.ClientMode, error) {
return getter.ClientModeDir, nil
}
func (g *gitGetter) Get(dst string, u *url.URL) error {
ctx := g.Context()
if _, err := exec.LookPath("git"); err != nil {
return fmt.Errorf("git must be available and on the PATH")
}
// The port number must be parseable as an integer. If not, the user
// was probably trying to use a scp-style address, in which case the
// ssh:// prefix must be removed to indicate that.
//
// This is not necessary in versions of Go which have patched
// CVE-2019-14809 (e.g. Go 1.12.8+)
if portStr := u.Port(); portStr != "" {
if _, err := strconv.ParseUint(portStr, 10, 16); err != nil {
return fmt.Errorf("invalid port number %q; if using the \"scp-like\" git address scheme where a colon introduces the path instead, remove the ssh:// portion and use just the git:: prefix", portStr)
}
}
// Extract some query parameters we use
var ref, sshKey string
var depth int
q := u.Query()
if len(q) > 0 {
ref = q.Get("ref")
q.Del("ref")
sshKey = q.Get("sshkey")
q.Del("sshkey")
if n, err := strconv.Atoi(q.Get("depth")); err == nil {
depth = n
}
q.Del("depth")
// Copy the URL
var newU url.URL = *u
u = &newU
u.RawQuery = q.Encode()
}
var sshKeyFile string
if sshKey != "" {
// Check that the git version is sufficiently new.
if err := checkGitVersion("2.3"); err != nil {
return fmt.Errorf("Error using ssh key: %v", err)
}
// We have an SSH key - decode it.
raw, err := base64.StdEncoding.DecodeString(sshKey)
if err != nil {
return err
}
// Create a temp file for the key and ensure it is removed.
fh, err := ioutil.TempFile("", "go-getter")
if err != nil {
return err
}
sshKeyFile = fh.Name()
defer os.Remove(sshKeyFile)
// Set the permissions prior to writing the key material.
if err := os.Chmod(sshKeyFile, 0600); err != nil {
return err
}
// Write the raw key into the temp file.
_, err = fh.Write(raw)
fh.Close()
if err != nil {
return err
}
}
// Clone or update the repository
_, err := os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
return err
}
if err == nil {
err = g.update(ctx, dst, sshKeyFile, ref, depth)
} else {
err = g.clone(ctx, dst, sshKeyFile, u, ref, depth)
}
if err != nil {
return err
}
// Next: check out the proper tag/branch if it is specified, and checkout
if ref != "" {
if err := g.checkout(dst, ref); err != nil {
return err
}
}
// Lastly, download any/all submodules.
return g.fetchSubmodules(ctx, dst, sshKeyFile, depth)
}
// GetFile for Git doesn't support updating at this time. It will download
// the file every time.
func (g *gitGetter) GetFile(dst string, u *url.URL) error {
td, tdcloser, err := safetemp.Dir("", "getter")
if err != nil {
return err
}
defer tdcloser.Close()
// Get the filename, and strip the filename from the URL so we can
// just get the repository directly.
filename := filepath.Base(u.Path)
u.Path = filepath.Dir(u.Path)
// Get the full repository
if err := g.Get(td, u); err != nil {
return err
}
// Copy the single file
u, err = urlhelper.Parse(fmtFileURL(filepath.Join(td, filename)))
if err != nil {
return err
}
fg := &getter.FileGetter{Copy: true}
return fg.GetFile(dst, u)
}
func (g *gitGetter) checkout(dst string, ref string) error {
cmd := exec.Command("git", "checkout", ref)
cmd.Dir = dst
return getRunCommand(cmd)
}
func (g *gitGetter) clone(ctx context.Context, dst, sshKeyFile string, u *url.URL, ref string, depth int) error {
args := []string{"clone"}
if ref == "" {
ref = findRemoteDefaultBranch(u)
}
if depth > 0 {
args = append(args, "--depth", strconv.Itoa(depth))
}
args = append(args, "--branch", ref, u.String(), dst)
cmd := exec.CommandContext(ctx, "git", args...)
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd)
}
func (g *gitGetter) update(ctx context.Context, dst, sshKeyFile, ref string, depth int) error {
// Determine if we're a branch. If we're NOT a branch, then we just
// switch to master prior to checking out
cmd := exec.CommandContext(ctx, "git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
cmd.Dir = dst
if getRunCommand(cmd) != nil {
// Not a branch, switch to default branch. This will also catch
// non-existent branches, in which case we want to switch to default
// and then checkout the proper branch later.
ref = findDefaultBranch(dst)
}
// We have to be on a branch to pull
if err := g.checkout(dst, ref); err != nil {
return err
}
if depth > 0 {
cmd = exec.Command("git", "pull", "--depth", strconv.Itoa(depth), "--ff-only")
} else {
cmd = exec.Command("git", "pull", "--ff-only")
}
cmd.Dir = dst
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd)
}
// fetchSubmodules downloads any configured submodules recursively.
func (g *gitGetter) fetchSubmodules(ctx context.Context, dst, sshKeyFile string, depth int) error {
args := []string{"submodule", "update", "--init", "--recursive"}
if depth > 0 {
args = append(args, "--depth", strconv.Itoa(depth))
}
cmd := exec.CommandContext(ctx, "git", args...)
cmd.Dir = dst
setupGitEnv(cmd, sshKeyFile)
return getRunCommand(cmd)
}
// findDefaultBranch checks the repo's origin remote for its default branch
// (generally "master"). "master" is returned if an origin default branch
// can't be determined.
func findDefaultBranch(dst string) string {
var stdoutbuf bytes.Buffer
cmd := exec.Command("git", "branch", "-r", "--points-at", "refs/remotes/origin/HEAD")
cmd.Dir = dst
cmd.Stdout = &stdoutbuf
err := cmd.Run()
matches := defaultBranchRegexp.FindStringSubmatch(stdoutbuf.String())
if err != nil || matches == nil {
return "master"
}
return matches[len(matches)-1]
}
// findRemoteDefaultBranch checks the remote repo's HEAD symref to return the remote repo's
// default branch. "master" is returned if no HEAD symref exists.
func findRemoteDefaultBranch(u *url.URL) string {
var stdoutbuf bytes.Buffer
cmd := exec.Command("git", "ls-remote", "--symref", u.String(), "HEAD")
cmd.Stdout = &stdoutbuf
err := cmd.Run()
matches := lsRemoteSymRefRegexp.FindStringSubmatch(stdoutbuf.String())
if err != nil || matches == nil {
return "master"
}
return matches[len(matches)-1]
}
// setupGitEnv sets up the environment for the given command. This is used to
// pass configuration data to git and ssh and enables advanced cloning methods.
func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
const gitSSHCommand = "GIT_SSH_COMMAND="
var sshCmd []string
// If we have an existing GIT_SSH_COMMAND, we need to append our options.
// We will also remove our old entry to make sure the behavior is the same
// with versions of Go < 1.9.
env := os.Environ()
for i, v := range env {
if strings.HasPrefix(v, gitSSHCommand) && len(v) > len(gitSSHCommand) {
sshCmd = []string{v}
env[i], env[len(env)-1] = env[len(env)-1], env[i]
env = env[:len(env)-1]
break
}
}
if len(sshCmd) == 0 {
sshCmd = []string{gitSSHCommand + "ssh"}
}
if sshKeyFile != "" {
// We have an SSH key temp file configured, tell ssh about this.
if runtime.GOOS == "windows" {
sshKeyFile = strings.Replace(sshKeyFile, `\`, `/`, -1)
}
sshCmd = append(sshCmd, "-i", sshKeyFile)
}
env = append(env, strings.Join(sshCmd, " "))
cmd.Env = env
}
// checkGitVersion is used to check the version of git installed on the system
// against a known minimum version. Returns an error if the installed version
// is older than the given minimum.
func checkGitVersion(min string) error {
want, err := version.NewVersion(min)
if err != nil {
return err
}
out, err := exec.Command("git", "version").Output()
if err != nil {
return err
}
fields := strings.Fields(string(out))
if len(fields) < 3 {
return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
}
v := fields[2]
if runtime.GOOS == "windows" && strings.Contains(v, ".windows.") {
// on windows, git version will return for example:
// git version 2.20.1.windows.1
// Which does not follow the semantic versionning specs
// https://semver.org. We remove that part in order for
// go-version to not error.
v = v[:strings.Index(v, ".windows.")]
}
have, err := version.NewVersion(v)
if err != nil {
return err
}
if have.LessThan(want) {
return fmt.Errorf("Required git version = %s, have %s", want, have)
}
return nil
}
// getRunCommand is a helper that will run a command and capture the output
// in the case an error happens.
func getRunCommand(cmd *exec.Cmd) error {
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Stderr = &buf
err := cmd.Run()
if err == nil {
return nil
}
if exiterr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
return fmt.Errorf(
"%s exited with %d: %s",
cmd.Path,
status.ExitStatus(),
buf.String())
}
}
return fmt.Errorf("error running %s: %s", cmd.Path, buf.String())
}

View File

@ -0,0 +1,724 @@
package getmodules
import (
"bytes"
"encoding/base64"
"io/ioutil"
"net/url"
"os"
"os/exec"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
getter "github.com/hashicorp/go-getter"
urlhelper "github.com/hashicorp/go-getter/helper/url"
)
var testHasGit bool
func init() {
if _, err := exec.LookPath("git"); err == nil {
testHasGit = true
}
}
func TestGitGetter_impl(t *testing.T) {
var _ getter.Getter = new(gitGetter)
}
func TestGitGetter(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "basic")
repo.commitFile("foo.txt", "hello")
// With a dir that doesn't exist
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath := filepath.Join(dst, "foo.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_branch(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "branch")
repo.git("checkout", "-b", "test-branch")
repo.commitFile("branch.txt", "branch")
q := repo.url.Query()
q.Add("ref", "test-branch")
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath := filepath.Join(dst, "branch.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
// Get again should work
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath = filepath.Join(dst, "branch.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_remoteWithoutMaster(t *testing.T) {
if !testHasGit {
t.Log("git not found, skipping")
t.Skip()
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "branch")
repo.git("checkout", "-b", "test-branch")
repo.commitFile("branch.txt", "branch")
q := repo.url.Query()
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath := filepath.Join(dst, "branch.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
// Get again should work
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath = filepath.Join(dst, "branch.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_shallowClone(t *testing.T) {
if !testHasGit {
t.Log("git not found, skipping")
t.Skip()
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "upstream")
repo.commitFile("upstream.txt", "0")
repo.commitFile("upstream.txt", "1")
// Specifiy a clone depth of 1
q := repo.url.Query()
q.Add("depth", "1")
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Assert rev-list count is '1'
cmd := exec.Command("git", "rev-list", "HEAD", "--count")
cmd.Dir = dst
b, err := cmd.Output()
if err != nil {
t.Fatalf("err: %s", err)
}
out := strings.TrimSpace(string(b))
if out != "1" {
t.Fatalf("expected rev-list count to be '1' but got %v", out)
}
}
func TestGitGetter_shallowCloneWithTag(t *testing.T) {
if !testHasGit {
t.Log("git not found, skipping")
t.Skip()
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "upstream")
repo.commitFile("v1.0.txt", "0")
repo.git("tag", "v1.0")
repo.commitFile("v1.1.txt", "1")
// Specifiy a clone depth of 1 with a tag
q := repo.url.Query()
q.Add("ref", "v1.0")
q.Add("depth", "1")
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Assert rev-list count is '1'
cmd := exec.Command("git", "rev-list", "HEAD", "--count")
cmd.Dir = dst
b, err := cmd.Output()
if err != nil {
t.Fatalf("err: %s", err)
}
out := strings.TrimSpace(string(b))
if out != "1" {
t.Fatalf("expected rev-list count to be '1' but got %v", out)
}
// Verify the v1.0 file exists
mainPath := filepath.Join(dst, "v1.0.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the v1.1 file does not exists
mainPath = filepath.Join(dst, "v1.1.txt")
if _, err := os.Stat(mainPath); err == nil {
t.Fatalf("expected v1.1 file to not exist")
}
}
func TestGitGetter_branchUpdate(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
// First setup the state with a fresh branch
repo := testGitRepo(t, "branch-update")
repo.git("checkout", "-b", "test-branch")
repo.commitFile("branch.txt", "branch")
// Get the "test-branch" branch
q := repo.url.Query()
q.Add("ref", "test-branch")
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath := filepath.Join(dst, "branch.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
// Commit an update to the branch
repo.commitFile("branch-update.txt", "branch-update")
// Get again should work
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath = filepath.Join(dst, "branch-update.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_tag(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
repo := testGitRepo(t, "tag")
repo.commitFile("tag.txt", "tag")
repo.git("tag", "v1.0")
q := repo.url.Query()
q.Add("ref", "v1.0")
repo.url.RawQuery = q.Encode()
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath := filepath.Join(dst, "tag.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
// Get again should work
if err := g.Get(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
mainPath = filepath.Join(dst, "tag.txt")
if _, err := os.Stat(mainPath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_GetFile(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempTestFile(t)
defer os.RemoveAll(filepath.Dir(dst))
repo := testGitRepo(t, "file")
repo.commitFile("file.txt", "hello")
// Download the file
repo.url.Path = filepath.Join(repo.url.Path, "file.txt")
if err := g.GetFile(dst, repo.url); err != nil {
t.Fatalf("err: %s", err)
}
// Verify the main file exists
if _, err := os.Stat(dst); err != nil {
t.Fatalf("err: %s", err)
}
assertContents(t, dst, "hello")
}
func TestGitGetter_gitVersion(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
if runtime.GOOS == "windows" {
t.Skip("skipping on windows since the test requires sh")
}
dir, err := ioutil.TempDir("", "go-getter")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
script := filepath.Join(dir, "git")
err = ioutil.WriteFile(
script,
[]byte("#!/bin/sh\necho \"git version 2.0 (Some Metadata Here)\n\""),
0700)
if err != nil {
t.Fatal(err)
}
defer func(v string) {
os.Setenv("PATH", v)
}(os.Getenv("PATH"))
os.Setenv("PATH", dir)
// Asking for a higher version throws an error
if err := checkGitVersion("2.3"); err == nil {
t.Fatal("expect git version error")
}
// Passes when version is satisfied
if err := checkGitVersion("1.9"); err != nil {
t.Fatal(err)
}
}
func TestGitGetter_sshKey(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
// avoid getting locked by a github authenticity validation prompt
os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
defer os.Setenv("GIT_SSH_COMMAND", "")
u, err := urlhelper.Parse("ssh://git@github.com/hashicorp/test-private-repo" +
"?sshkey=" + encodedKey)
if err != nil {
t.Fatal(err)
}
if err := g.Get(dst, u); err != nil {
t.Fatalf("err: %s", err)
}
readmePath := filepath.Join(dst, "README.md")
if _, err := os.Stat(readmePath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_sshSCPStyle(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
// avoid getting locked by a github authenticity validation prompt
os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
defer os.Setenv("GIT_SSH_COMMAND", "")
// This test exercises the combination of the git detector and the
// git getter, to make sure that together they make scp-style URLs work.
client := &getter.Client{
Src: "git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey,
Dst: dst,
Pwd: ".",
Mode: getter.ClientModeDir,
Detectors: []getter.Detector{
new(getter.GitDetector),
},
Getters: map[string]getter.Getter{
"git": g,
},
}
if err := client.Get(); err != nil {
t.Fatalf("client.Get failed: %s", err)
}
readmePath := filepath.Join(dst, "README.md")
if _, err := os.Stat(readmePath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_sshExplicitPort(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
// avoid getting locked by a github authenticity validation prompt
os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
defer os.Setenv("GIT_SSH_COMMAND", "")
// This test exercises the combination of the git detector and the
// git getter, to make sure that together they make scp-style URLs work.
client := &getter.Client{
Src: "git::ssh://git@github.com:22/hashicorp/test-private-repo?sshkey=" + encodedKey,
Dst: dst,
Pwd: ".",
Mode: getter.ClientModeDir,
Detectors: []getter.Detector{
new(getter.GitDetector),
},
Getters: map[string]getter.Getter{
"git": g,
},
}
if err := client.Get(); err != nil {
t.Fatalf("client.Get failed: %s", err)
}
readmePath := filepath.Join(dst, "README.md")
if _, err := os.Stat(readmePath); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestGitGetter_sshSCPStyleInvalidScheme(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
encodedKey := base64.StdEncoding.EncodeToString([]byte(testGitToken))
// avoid getting locked by a github authenticity validation prompt
os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
defer os.Setenv("GIT_SSH_COMMAND", "")
// This test exercises the combination of the git detector and the
// git getter, to make sure that together they make scp-style URLs work.
client := &getter.Client{
Src: "git::ssh://git@github.com:hashicorp/test-private-repo?sshkey=" + encodedKey,
Dst: dst,
Pwd: ".",
Mode: getter.ClientModeDir,
Detectors: []getter.Detector{
new(getter.GitDetector),
},
Getters: map[string]getter.Getter{
"git": g,
},
}
err := client.Get()
if err == nil {
t.Fatalf("get succeeded; want error")
}
got := err.Error()
want1, want2 := `invalid source string`, `invalid port number "hashicorp"`
if !(strings.Contains(got, want1) || strings.Contains(got, want2)) {
t.Fatalf("wrong error\ngot: %s\nwant: %q or %q", got, want1, want2)
}
}
func TestGitGetter_submodule(t *testing.T) {
if !testHasGit {
t.Skip("git not found, skipping")
}
g := new(gitGetter)
dst := tempDir(t)
relpath := func(basepath, targpath string) string {
relpath, err := filepath.Rel(basepath, targpath)
if err != nil {
t.Fatal(err)
}
return strings.Replace(relpath, `\`, `/`, -1)
// on windows git still prefers relatives paths
// containing `/` for submodules
}
// Set up the grandchild
gc := testGitRepo(t, "grandchild")
gc.commitFile("grandchild.txt", "grandchild")
// Set up the child
c := testGitRepo(t, "child")
c.commitFile("child.txt", "child")
c.git("submodule", "add", "-f", relpath(c.dir, gc.dir))
c.git("commit", "-m", "Add grandchild submodule")
// Set up the parent
p := testGitRepo(t, "parent")
p.commitFile("parent.txt", "parent")
p.git("submodule", "add", "-f", relpath(p.dir, c.dir))
p.git("commit", "-m", "Add child submodule")
// Clone the root repository
if err := g.Get(dst, p.url); err != nil {
t.Fatalf("err: %s", err)
}
// Check that the files exist
for _, path := range []string{
filepath.Join(dst, "parent.txt"),
filepath.Join(dst, "child", "child.txt"),
filepath.Join(dst, "child", "grandchild", "grandchild.txt"),
} {
if _, err := os.Stat(path); err != nil {
t.Fatalf("err: %s", err)
}
}
}
func TestGitGetter_setupGitEnv_sshKey(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("skipping on windows since the test requires sh")
}
cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND")
setupGitEnv(cmd, "/tmp/foo.pem")
out, err := cmd.Output()
if err != nil {
t.Fatal(err)
}
actual := strings.TrimSpace(string(out))
if actual != "ssh -i /tmp/foo.pem" {
t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual)
}
}
func TestGitGetter_setupGitEnvWithExisting_sshKey(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skipf("skipping on windows since the test requires sh")
return
}
// start with an existing ssh command configuration
os.Setenv("GIT_SSH_COMMAND", "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes")
defer os.Setenv("GIT_SSH_COMMAND", "")
cmd := exec.Command("/bin/sh", "-c", "echo $GIT_SSH_COMMAND")
setupGitEnv(cmd, "/tmp/foo.pem")
out, err := cmd.Output()
if err != nil {
t.Fatal(err)
}
actual := strings.TrimSpace(string(out))
if actual != "ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i /tmp/foo.pem" {
t.Fatalf("unexpected GIT_SSH_COMMAND: %q", actual)
}
}
// gitRepo is a helper struct which controls a single temp git repo.
type gitRepo struct {
t *testing.T
url *url.URL
dir string
}
// testGitRepo creates a new test git repository.
func testGitRepo(t *testing.T, name string) *gitRepo {
t.Helper()
dir, err := ioutil.TempDir("", "go-getter")
if err != nil {
t.Fatal(err)
}
dir = filepath.Join(dir, name)
if err := os.Mkdir(dir, 0700); err != nil {
t.Fatal(err)
}
r := &gitRepo{
t: t,
dir: dir,
}
url, err := urlhelper.Parse("file://" + r.dir)
if err != nil {
t.Fatal(err)
}
r.url = url
t.Logf("initializing git repo in %s", dir)
r.git("init")
r.git("config", "user.name", "go-getter")
r.git("config", "user.email", "go-getter@hashicorp.com")
return r
}
// git runs a git command against the repo.
func (r *gitRepo) git(args ...string) {
cmd := exec.Command("git", args...)
cmd.Dir = r.dir
bfr := bytes.NewBuffer(nil)
cmd.Stderr = bfr
if err := cmd.Run(); err != nil {
r.t.Fatal(err, bfr.String())
}
}
// commitFile writes and commits a text file to the repo.
func (r *gitRepo) commitFile(file, content string) {
path := filepath.Join(r.dir, file)
if err := ioutil.WriteFile(path, []byte(content), 0600); err != nil {
r.t.Fatal(err)
}
r.git("add", file)
r.git("commit", "-m", "Adding "+file)
}
// This is a read-only deploy key for an empty test repository.
// Note: This is split over multiple lines to avoid being disabled by key
// scanners automatically.
var testGitToken = `-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEA9cHsxCl3Jjgu9DHpwvmfFOl1XEdY+ShHDR/cMnzJ5ddk5/oV
Wy6EWatvyHZfRSZMwzv4PtKeUPm6iXjqWp4xdWU9khlPzozyj+U9Fq70TRVUW9E5
T1XdQVwJE421yffr4VMMwu60wBqjI1epapH2i2inYvw9Zl9X2MXq0+jTvFvDerbT
mDtfStDPljenELAIZtWVETSvbI46gALwbxbM2292ZUIL4D6jRz0aZMmyy/twYv8r
9WGJLwmYzU518Ie7zqKW/mCTdTrV0WRiDj0MeRaPgrGY9amuHE4r9iG/cJkwpKAO
Ccz0Hs6i89u9vZnTqZU9V7weJqRAQcMjXXR6yQIDAQABAoIBAQDBzICKnGxiTlHw
rd+6qqChnAy5jWYDbZjCJ8q8YZ3RS08+g/8NXZxvHftTqM0uOaq1FviHig3gq15H
hHvCpBc6jXDFYoKFzq6FfO/0kFkE5HoWweIgxwRow0xBCDJAJ+ryUEyy+Ay/pQHb
IAjwilRS0V+WdnVw4mTjBAhPvb4jPOo97Yfy3PYUyx2F3newkqXOZy+zx3G/ANoa
ncypfMGyy76sfCWKqw4J1gVkVQLwbB6gQkXUFGYwY9sRrxbG93kQw76Flc/E/s52
62j4v1IM0fq0t/St+Y/+s6Lkw` + `aqt3ft1nsqWcRaVDdqvMfkzgJGXlw0bGzJG5MEQ
AIBq3dHRAoGBAP8OeG/DKG2Z1VmSfzuz1pas1fbZ+F7venOBrjez3sKlb3Pyl2aH
mt2wjaTUi5v10VrHgYtOEdqyhQeUSYydWXIBKNMag0NLLrfFUKZK+57wrHWFdFjn
VgpsdkLSNTOZpC8gA5OaJ+36IcOPfGqyyP9wuuRoaYnVT1KEzqLa9FEFAoGBAPaq
pglwhil2rxjJE4zq0afQLNpAfi7Xqcrepij+xvJIcIj7nawxXuPxqRFxONE/h3yX
zkybO8wLdbHX9Iw/wc1j50Uf1Z5gHdLf7/hQJoWKpz1RnkWRy6CYON8v1tpVp0tb
OAajR/kZnzebq2mfa7pyy5zDCX++2kp/dcFwHf31AoGAE8oupBVTZLWj7TBFuP8q
LkS40U92Sv9v09iDCQVmylmFvUxcXPM2m+7f/qMTNgWrucxzC7kB/6MMWVszHbrz
vrnCTibnemgx9sZTjKOSxHFOIEw7i85fSa3Cu0qOIDPSnmlwfZpfcMKQrhjLAYhf
uhooFiLX1X78iZ2OXup4PHUCgYEAsmBrm83sp1V1gAYBBlnVbXakyNv0pCk/Vz61
iFXeRt1NzDGxLxGw3kQnED8BaIh5kQcyn8Fud7sdzJMv/LAqlT4Ww60mzNYTGyjo
H3jOsqm3ESfRvduWFreeAQBWbiOczGjV1i8D4EbAFfWT+tjXjchwKBf+6Yt5zn/o
Bw/uEHUCgYAFs+JPOR25oRyBs7ujrMo/OY1z/eXTVVgZxY+tYGe1FJqDeFyR7ytK
+JBB1MuDwQKGm2wSIXdCzTNoIx2B9zTseiPTwT8G7vqNFhXoIaTBp4P2xIQb45mJ
7GkTsMBHwpSMOXgX9Weq3v5xOJ2WxVtjENmd6qzxcYCO5lP15O17hA==
-----END RSA PRIVATE KEY-----`
func assertContents(t *testing.T, path string, contents string) {
data, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("err: %s", err)
}
if !reflect.DeepEqual(data, []byte(contents)) {
t.Fatalf("bad. expected:\n\n%s\n\nGot:\n\n%s", contents, string(data))
}
}
func tempDir(t *testing.T) string {
dir, err := ioutil.TempDir("", "tf")
if err != nil {
t.Fatalf("err: %s", err)
}
if err := os.RemoveAll(dir); err != nil {
t.Fatalf("err: %s", err)
}
return dir
}
func tempTestFile(t *testing.T) string {
dir := tempDir(t)
return filepath.Join(dir, "foo")
}