tools/terraform-bundle: refactor to use new provider installer and provider directory layouts (#24629)

* tools/terraform-bundle: refactor to use new provider installer and
provider directory layouts

terraform-bundle now supports a "source" attribute for providers,
uses the new provider installer, and the archive it creates preserves
the new (required) directory hierarchy for providers, under a "plugins"
directory.

This is a breaking change in many ways: source is required for any
non-HashiCorp provider, locally-installed providers must be given a
source (can be arbitrary, see docs) and placed in the expected directory
hierarchy, and the unzipped archive is no longer flat; there is a new
"plugins" directory created with providers in the new directory layout.

This PR also extends the existing test to check the contents of the zip
file.

TODO: Re-enable e2e tests (currently suppressed with a t.Skip)
This commit includes an update to our travis configuration, so the terraform-bundle e2e tests run. It also turns off the e2e tests, which will fail until we have a terraform 0.13.* release under releases.hashicorp.com. We decided it was better to merge this now instead of waiting when we started seeing issues opened from users who built terraform-bundle from 0.13 and found it didn't work with 0.12 - better that they get an immediate error message from the binary directing them to build from the appropriate release.

.circleci/config.yml

@@ -81,10 +81,7 @@ jobs:
       - run:
           name: Run Go E2E Tests
           command: |
-            PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
+            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest ./tools/terraform-bundle/e2etest
-            echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
-            echo $PACKAGE_NAMES
-            gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./command/e2etest
 
       # save coverage report parts
       - persist_to_workspace:

e2e/e2e.go

@@ -270,3 +270,8 @@ func GoBuild(pkgPath, tmpPrefix string) string {
 
 	return tmpFilename
 }
+
+// WorkDir() returns the binary workdir
+func (b *binary) WorkDir() string {
+	return b.workDir
+}

internal/providercache/dir.go

@@ -55,13 +55,13 @@ func NewDir(baseDir string) *Dir {
 	}
 }
 
-// newDirWithPlatform is a variant of NewDir that allows selecting a specific
+// NewDirWithPlatform is a variant of NewDir that allows selecting a specific
 // target platform, rather than taking the current one where this code is
 // running.
 //
 // This is primarily intended for portable unit testing and not particularly
-// useful in "real" callers.
+// useful in "real" callers, with the exception of terraform-bundle.
-func newDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
+func NewDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir {
 	return &Dir{
 		baseDir:        baseDir,
 		targetPlatform: platform,

internal/providercache/dir_modify_test.go

@@ -84,8 +84,8 @@ func TestLinkFromOtherCache(t *testing.T) {
 		addrs.DefaultRegistryHost, "hashicorp", "null",
 	)
 
-	srcDir := newDirWithPlatform(srcDirPath, windowsPlatform)
+	srcDir := NewDirWithPlatform(srcDirPath, windowsPlatform)
-	tmpDir := newDirWithPlatform(tmpDirPath, windowsPlatform)
+	tmpDir := NewDirWithPlatform(tmpDirPath, windowsPlatform)
 
 	// First we'll check our preconditions: srcDir should have only the
 	// null provider version 2.0.0 in it, because we're faking that we're on

internal/providercache/dir_test.go

@@ -40,7 +40,7 @@ func TestDirReading(t *testing.T) {
 
 	t.Run("ProviderLatestVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			got := dir.ProviderLatestVersion(nullProvider)
 			want := &CachedProvider{
@@ -59,7 +59,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// random provider is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderLatestVersion(randomProvider)
@@ -70,7 +70,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderLatestVersion(nonExistProvider)
@@ -84,7 +84,7 @@ func TestDirReading(t *testing.T) {
 
 	t.Run("ProviderVersion", func(t *testing.T) {
 		t.Run("exists", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("2.0.0"))
 			want := &CachedProvider{
@@ -100,7 +100,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("specified version is not cached", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// there is no v5.0.0 package in our fixtures dir
 			got := dir.ProviderVersion(nullProvider, versions.MustParseVersion("5.0.0"))
@@ -111,7 +111,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no package for current platform", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// random provider 1.2.0 is only cached for linux_amd64 in our fixtures dir
 			got := dir.ProviderVersion(randomProvider, versions.MustParseVersion("1.2.0"))
@@ -122,7 +122,7 @@ func TestDirReading(t *testing.T) {
 			}
 		})
 		t.Run("no versions available at all", func(t *testing.T) {
-			dir := newDirWithPlatform(testDir, windowsPlatform)
+			dir := NewDirWithPlatform(testDir, windowsPlatform)
 
 			// nonexist provider is not present in our fixtures dir at all
 			got := dir.ProviderVersion(nonExistProvider, versions.MustParseVersion("1.0.0"))
@@ -135,7 +135,7 @@ func TestDirReading(t *testing.T) {
 	})
 
 	t.Run("AllAvailablePackages", func(t *testing.T) {
-		dir := newDirWithPlatform(testDir, linuxPlatform)
+		dir := NewDirWithPlatform(testDir, linuxPlatform)
 
 		got := dir.AllAvailablePackages()
 		want := map[addrs.Provider][]CachedProvider{

tools/terraform-bundle/README.md

@@ -34,12 +34,8 @@ the rest of this README to be in `PATH`.
 
 `terraform-bundle` is a repackaging of the module installation functionality
 from Terraform itself, so for best results you should build from the tag
-relating to the version of Terraform you plan to use. There is some slack in
+relating to the version of Terraform you plan to use. For example, use the v0.12
-this requirement due to the fact that the module installation behavior changes
+tag to build a version of terraform-bundle compatible with Terraform v0.12*.
-rarely, but please note that in particular bundles for versions of
-Terraform before v0.12 must be built from a `terraform-bundle` built against
-a Terraform v0.11 tag at the latest, since Terraform v0.12 installs plugins
-in a different way that is not compatible.
 
 ## Usage
 
@@ -59,19 +55,26 @@ terraform {
 # Define which provider plugins are to be included
 providers {
   # Include the newest "aws" provider version in the 1.0 series.
-  aws = ["~> 1.0"]
+  aws = {
+    versions = ["~> 1.0"]
+  }
 
   # Include both the newest 1.0 and 2.0 versions of the "google" provider.
   # Each item in these lists allows a distinct version to be added. If the
   # two expressions match different versions then _both_ are included in
   # the bundle archive.
-  google = ["~> 1.0", "~> 2.0"]
+  google = {
+    versions = ["~> 1.0", "~> 2.0"]
+  }
 
   # Include a custom plugin to the bundle. Will search for the plugin in the
-  # plugins directory, and package it with the bundle archive. Plugin must have
+  # plugins directory and package it with the bundle archive. Plugin must have
-  # a name of the form: terraform-provider-*, and must be build with the operating
+  # a name of the form: terraform-provider-*, and must be built with the operating
-  # system and architecture that terraform enterprise is running, e.g. linux and amd64
+  # system and architecture that terraform enterprise is running, e.g. linux and amd64.
-  customplugin = ["0.1"]
+  customplugin = {
+    versions = ["0.1"]
+    source = "myorg/customplugin"
+  }
 }
 
 ```
@@ -80,10 +83,10 @@ The `terraform` block defines which version of Terraform will be included
 in the bundle. An exact version is required here.
 
 The `providers` block defines zero or more providers to include in the bundle
-along with core Terraform. Each attribute in this block is a provider name,
+along with core Terraform. Each attribute is a provider name, and its value is a
-and its value is a list of version constraints. For each given constraint,
+block with the list of version constraints and (optional) source. For each given
-`terraform-bundle` will find the newest available version matching the
+constraint, `terraform-bundle` will find the newest available version matching
-constraint and include it in the bundle.
+the constraint and include it in the bundle.
 
 It is allowed to specify multiple constraints for the same provider, in which
 case multiple versions can be included in the resulting bundle. Each constraint
@@ -119,13 +122,44 @@ this composite version number so that bundle archives can be easily
 distinguished from official release archives and from each other when multiple
 bundles contain the same core Terraform version.
 
-To include custom plugins in the bundle file, create a local directory "./plugins"
+## Custom Plugins
-and put all the plugins you want to include there. Optionally, you can use the
+To include custom plugins in the bundle file, create a local directory
-`-plugin-dir` flag to specify a location where to find the plugins. To be recognized
+"./plugins" and put all the plugins you want to include there, under the
-as a valid plugin, the file must have a name of the form
+required [sub directory](#plugins-directory-layout). Optionally, you can use the
-`terraform-provider-<NAME>_v<VERSION>`. In
+`-plugin-dir` flag to specify a location where to find the plugins. To be
-addition, ensure that the plugin is built using the same operating system and
+recognized as a valid plugin, the file must have a name of the form
-architecture used for Terraform Enterprise. Typically this will be `linux` and `amd64`.
+`terraform-provider-<NAME>`. In addition, ensure that the plugin is built using
+the same operating system and architecture used for Terraform Enterprise.
+Typically this will be `linux` and `amd64`.
+
+### Plugins Directory Layout
+To include custom plugins in the bundle file, you must specify a "source"
+attribute in the configuration and place the plugin in the appropriate
+subdirectory under "./plugins". The directory must have the following layout:
+
+```
+./plugins/$SOURCEHOST/$SOURCENAMESPACE/$NAME/$VERSION/$OS_$ARCH/
+```
+
+When installing custom plugins, you may choose any arbitrary identifier for the
+$SOURCEHOST and $SOURCENAMESPACE subdirectories. 
+
+For example, given the following configuration and a plugin built for Terraform Enterprise:
+
+```
+providers {
+  customplugin = {
+    versions = ["0.1"]
+    source = "example.com/myorg/customplugin"
+  }
+}
+```
+
+The binary must be placed in the following directory:
+
+```
+./plugins/example.com/myorg/customplugin/0.1/linux_amd64/
+```
 
 ## Provider Resolution Behavior
 

tools/terraform-bundle/config.go

@@ -5,20 +5,27 @@ import (
 	"io/ioutil"
 
 	"github.com/hashicorp/hcl"
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
 	"github.com/hashicorp/terraform/plugin/discovery"
 )
 
-var zeroTwelve = discovery.ConstraintStr(">= 0.12.0").MustParse()
+var zeroThirteen = discovery.ConstraintStr(">= 0.13.0").MustParse()
 
 type Config struct {
-	Terraform TerraformConfig                      `hcl:"terraform"`
+	Terraform TerraformConfig           `hcl:"terraform"`
-	Providers map[string][]discovery.ConstraintStr `hcl:"providers"`
+	Providers map[string]ProviderConfig `hcl:"providers"`
 }
 
 type TerraformConfig struct {
 	Version discovery.VersionStr `hcl:"version"`
 }
 
+type ProviderConfig struct {
+	Versions []string `hcl:"versions"`
+	Source   string   `hcl:"source"`
+}
+
 func LoadConfig(src []byte, filename string) (*Config, error) {
 	config := &Config{}
 	err := hcl.Decode(config, string(src))
@@ -49,17 +56,23 @@ func (c *Config) validate() error {
 	if v, err = c.Terraform.Version.Parse(); err != nil {
 		return fmt.Errorf("terraform.version: %s", err)
 	}
-	if !zeroTwelve.Allows(v) {
+	if !zeroThirteen.Allows(v) {
-		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.12 and later; build terraform-bundle from the v0.11 branch or a v0.11.* tag to construct bundles for earlier versions")
+		return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.13 and later; build terraform-bundle from a release tag (such as v0.12.*) to construct bundles for earlier versions")
 	}
 
 	if c.Providers == nil {
-		c.Providers = map[string][]discovery.ConstraintStr{}
+		c.Providers = map[string]ProviderConfig{}
 	}
 
 	for k, cs := range c.Providers {
-		for _, c := range cs {
+		if cs.Source != "" {
-			if _, err := c.Parse(); err != nil {
+			_, diags := addrs.ParseProviderSourceString(cs.Source)
+			if diags.HasErrors() {
+				return fmt.Errorf("providers.%s: %s", k, diags.Err().Error())
+			}
+		}
+		for _, c := range cs.Versions {
+			if _, err := getproviders.ParseVersionConstraints(c); err != nil {
 				return fmt.Errorf("providers.%s: %s", k, err)
 			}
 		}

tools/terraform-bundle/e2etest/package_test.go

@@ -1,7 +1,11 @@
 package e2etest
 
 import (
+	"archive/zip"
+	"fmt"
+	"io/ioutil"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -10,6 +14,9 @@ import (
 
 func TestPackage_empty(t *testing.T) {
 	t.Parallel()
+	// The e2etests can be reenabled when there is a terraform v0.13* release
+	// available on releases.hashicorp.com.
+	t.Skip("terraform-bundle e2e tests are temporarily paused")
 
 	// This test reaches out to releases.hashicorp.com to download the
 	// template provider, so it can only run if network access is allowed.
@@ -45,11 +52,15 @@ func TestPackage_empty(t *testing.T) {
 func TestPackage_manyProviders(t *testing.T) {
 	t.Parallel()
 
-	// This test reaches out to releases.hashicorp.com to download the
+	// The e2etests can be reenabled when there is a terraform v0.13* release
-	// template provider, so it can only run if network access is allowed.
+	// available on releases.hashicorp.com.
-	// We intentionally don't try to stub this here, because there's already
+	t.Skip("terraform-bundle e2e tests are temporarily paused")
-	// a stubbed version of this in the "command" package and so the goal here
+
-	// is to test the interaction with the real repository.
+	// This test reaches out to releases.hashicorp.com to download providers, so
+	// it can only run if network access is allowed. We intentionally don't try
+	// to stub this here, because there's already a stubbed version of this in
+	// the "command" package and so the goal here is to test the interaction
+	// with the real repository.
 	skipIfCannotAccessNetwork(t)
 
 	fixturePath := filepath.Join("testdata", "many-providers")
@@ -68,27 +79,22 @@ func TestPackage_manyProviders(t *testing.T) {
 	// Here we have to check each provider separately
 	// because it's internally held in a map (i.e. not guaranteed order)
 
-	if !strings.Contains(stdout, `- Resolving "aws" provider (~> 2.26.0)...
+	if !strings.Contains(stdout, `- Finding hashicorp/aws versions matching "~> 2.26.0"...
-- Checking for provider plugin on https://releases.hashicorp.com...
+- Installing hashicorp/aws v2.26.0...`) {
-- Downloading plugin for provider "aws" (hashicorp/aws) 2.26.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
 
-	if !strings.Contains(stdout, `- Resolving "kubernetes" provider (1.8.0)...
+	if !strings.Contains(stdout, `- Finding hashicorp/kubernetes versions matching "1.8.0"...
-- Checking for provider plugin on https://releases.hashicorp.com...
+- Installing hashicorp/kubernetes v1.8.0...
-- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.8.0...
+- Finding hashicorp/kubernetes versions matching "1.8.1"...
-- Resolving "kubernetes" provider (1.8.1)...
+- Installing hashicorp/kubernetes v1.8.1...
-- Checking for provider plugin on https://releases.hashicorp.com...
+- Finding hashicorp/kubernetes versions matching "1.9.0"...
-- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.8.1...
+- Installing hashicorp/kubernetes v1.9.0...`) {
-- Resolving "kubernetes" provider (1.9.0)...
-- Checking for provider plugin on https://releases.hashicorp.com...
-- Downloading plugin for provider "kubernetes" (hashicorp/kubernetes) 1.9.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
 
-	if !strings.Contains(stdout, `- Resolving "null" provider (2.1.0)...
+	if !strings.Contains(stdout, `- Finding hashicorp/null versions matching "2.1.0"...
-- Checking for provider plugin on https://releases.hashicorp.com...
+- Installing hashicorp/null v2.1.0...`) {
-- Downloading plugin for provider "null" (hashicorp/null) 2.1.0...`) {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
 
@@ -101,4 +107,129 @@ func TestPackage_manyProviders(t *testing.T) {
 	if !strings.Contains(stdout, "All done!") {
 		t.Errorf("success message is missing from output:\n%s", stdout)
 	}
+
+	// check the contents of the created zipfile
+	files, err := ioutil.ReadDir(tfBundle.WorkDir())
+	if err != nil {
+		t.Fatalf("error reading workdir: %s", err)
+	}
+	for _, file := range files {
+		if strings.Contains(file.Name(), "terraform_0.12.0-bundle") {
+			read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name()))
+			if err != nil {
+				t.Fatalf("Failed to open archive: %s", err)
+			}
+			defer read.Close()
+
+			expectedFiles := map[string]struct{}{
+				"terraform":                                   {},
+				testProviderBinaryPath("null", "2.1.0"):       {},
+				testProviderBinaryPath("aws", "2.26.0"):       {},
+				testProviderBinaryPath("kubernetes", "1.8.0"): {},
+				testProviderBinaryPath("kubernetes", "1.8.1"): {},
+				testProviderBinaryPath("kubernetes", "1.9.0"): {},
+			}
+			extraFiles := make(map[string]struct{})
+
+			for _, file := range read.File {
+				if _, exists := expectedFiles[file.Name]; exists {
+					delete(expectedFiles, file.Name)
+				} else {
+					extraFiles[file.Name] = struct{}{}
+				}
+			}
+			if len(expectedFiles) != 0 {
+				t.Errorf("missing expected file(s): %#v", expectedFiles)
+			}
+			if len(extraFiles) != 0 {
+				t.Errorf("found extra unexpected file(s): %#v", extraFiles)
+			}
+		}
+	}
+}
+
+func TestPackage_localProviders(t *testing.T) {
+	t.Parallel()
+
+	// The e2etests can be reenabled when there is a terraform v0.13* release
+	// available on releases.hashicorp.com.
+	t.Skip("terraform-bundle e2e tests are temporarily paused")
+
+	// This test reaches out to releases.hashicorp.com to download terrafrom, so
+	// it can only run if network access is allowed. The providers are installed
+	// from the local cache.
+	skipIfCannotAccessNetwork(t)
+
+	fixturePath := filepath.Join("testdata", "local-providers")
+	tfBundle := e2e.NewBinary(bundleBin, fixturePath)
+	defer tfBundle.Close()
+
+	// we explicitly specify the platform so that tests can find the local binary under the expected directory
+	stdout, stderr, err := tfBundle.Run("package", "-os=darwin", "-arch=amd64", "terraform-bundle.hcl")
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	// Here we have to check each provider separately
+	// because it's internally held in a map (i.e. not guaranteed order)
+	if !strings.Contains(stdout, "Fetching Terraform 0.12.0 core package...") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "Creating terraform_0.12.0-bundle") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+	if !strings.Contains(stdout, "All done!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	// check the contents of the created zipfile
+	files, err := ioutil.ReadDir(tfBundle.WorkDir())
+	if err != nil {
+		t.Fatalf("error reading workdir: %s", err)
+	}
+	for _, file := range files {
+		if strings.Contains(file.Name(), "terraform_0.12.0-bundle") {
+			read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name()))
+			if err != nil {
+				t.Fatalf("Failed to open archive: %s", err)
+			}
+			defer read.Close()
+
+			expectedFiles := map[string]struct{}{
+				"terraform": {},
+				"plugins/example.com/myorg/mycloud/0.1.0/darwin_amd64/terraform-provider-mycloud": {},
+			}
+			extraFiles := make(map[string]struct{})
+
+			for _, file := range read.File {
+				if _, exists := expectedFiles[file.Name]; exists {
+					delete(expectedFiles, file.Name)
+				} else {
+					extraFiles[file.Name] = struct{}{}
+				}
+			}
+			if len(expectedFiles) != 0 {
+				t.Errorf("missing expected file(s): %#v", expectedFiles)
+			}
+			if len(extraFiles) != 0 {
+				t.Errorf("found extra unexpected file(s): %#v", extraFiles)
+			}
+		}
+	}
+}
+
+// testProviderBinaryPath takes a provider name (assumed to be a hashicorp
+// provider) and version and returns the expected binary path, relative to the
+// archive, for the plugin.
+func testProviderBinaryPath(provider, version string) string {
+	os := runtime.GOOS
+	arch := runtime.GOARCH
+	return fmt.Sprintf(
+		"plugins/registry.terraform.io/hashicorp/%s/%s/%s_%s/terraform-provider-%s_v%s_x4",
+		provider, version, os, arch, provider, version,
+	)
 }

tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl

@@ -1,3 +1,3 @@
 terraform {
-  version = "0.12.0"
+  version = "0.13.0"
 }

tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud

@@ -0,0 +1 @@
+I am a fake binary.

tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl

@@ -0,0 +1,11 @@
+terraform {
+  version = "0.3.0"
+}
+
+providers {
+  // this provider is installed in .plugins
+  mycloud = {
+    versions = ["0.1"]
+    source   = "example.com/myorg/mycloud"
+  }
+}

tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl

@@ -1,9 +1,17 @@
 terraform {
-  version = "0.12.0"
+  version = "0.13.0"
 }
 
 providers {
-  aws        = ["~> 2.26.0"]
+  aws = {
-  kubernetes = ["1.8.0", "1.8.1", "1.9.0"]
+    versions = ["~> 2.26.0"]
-  null       = ["2.1.0"]
+  }
+
+  kubernetes = {
+    versions = ["1.8.0", "1.8.1", "1.9.0"]
+  }
+
+  null = {
+    versions = ["2.1.0"]
+  }
 }

tools/terraform-bundle/package.go

@@ -2,85 +2,37 @@ package main
 
 import (
 	"archive/zip"
+	"context"
+	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
-	"flag"
-
-	"io"
-
 	getter "github.com/hashicorp/go-getter"
+	"github.com/hashicorp/terraform-svchost/disco"
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/httpclient"
+	"github.com/hashicorp/terraform/internal/getproviders"
+	"github.com/hashicorp/terraform/internal/providercache"
 	discovery "github.com/hashicorp/terraform/plugin/discovery"
+	"github.com/hashicorp/terraform/tfdiags"
+	"github.com/hashicorp/terraform/version"
 	"github.com/mitchellh/cli"
 )
 
 var releaseHost = "https://releases.hashicorp.com"
 
+var pluginDir = ".plugins"
+
 type PackageCommand struct {
 	ui cli.Ui
 }
 
-// shameless stackoverflow copy + pasta https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
-func CopyFile(src, dst string) (err error) {
-	sfi, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	if !sfi.Mode().IsRegular() {
-		// cannot copy non-regular files (e.g., directories,
-		// symlinks, devices, etc.)
-		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
-	}
-	dfi, err := os.Stat(dst)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return
-		}
-	} else {
-		if !(dfi.Mode().IsRegular()) {
-			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
-		}
-		if os.SameFile(sfi, dfi) {
-			return
-		}
-	}
-	if err = os.Link(src, dst); err == nil {
-		return
-	}
-	err = copyFileContents(src, dst)
-	os.Chmod(dst, sfi.Mode())
-	return
-}
-
-// see above
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		cerr := out.Close()
-		if err == nil {
-			err = cerr
-		}
-	}()
-	if _, err = io.Copy(out, in); err != nil {
-		return
-	}
-	err = out.Sync()
-	return
-}
-
 func (c *PackageCommand) Run(args []string) int {
 	flags := flag.NewFlagSet("package", flag.ExitOnError)
 	osPtr := flags.String("os", "", "Target operating system")
@@ -94,7 +46,6 @@ func (c *PackageCommand) Run(args []string) int {
 
 	osName := runtime.GOOS
 	archName := runtime.GOARCH
-	pluginDir := "./plugins"
 	if *osPtr != "" {
 		osName = *osPtr
 	}
@@ -117,86 +68,102 @@ func (c *PackageCommand) Run(args []string) int {
 		return 1
 	}
 
-	if discovery.ConstraintStr("< 0.10.0-beta1").MustParse().Allows(config.Terraform.Version.MustParse()) {
+	tmpDir, err := ioutil.TempDir("", "terraform-bundle")
-		c.ui.Error("Bundles can be created only for Terraform 0.10 or newer")
-		return 1
-	}
-
-	workDir, err := ioutil.TempDir("", "terraform-bundle")
 	if err != nil {
 		c.ui.Error(fmt.Sprintf("Could not create temporary dir: %s", err))
 		return 1
 	}
+	// symlinked tmp directories can cause odd behaviors.
+	workDir, err := filepath.EvalSymlinks(tmpDir)
 	defer os.RemoveAll(workDir)
 
 	c.ui.Info(fmt.Sprintf("Fetching Terraform %s core package...", config.Terraform.Version))
 
 	coreZipURL := c.coreURL(config.Terraform.Version, osName, archName)
 	err = getter.Get(workDir, coreZipURL)
-
 	if err != nil {
 		c.ui.Error(fmt.Sprintf("Failed to fetch core package from %s: %s", coreZipURL, err))
 		return 1
 	}
 
-	c.ui.Info(fmt.Sprintf("Fetching 3rd party plugins in directory: %s", pluginDir))
+	// get the list of required providers from the config
-	dirs := []string{pluginDir} //FindPlugins requires an array
+	reqs := make(map[addrs.Provider][]string)
-	localPlugins := discovery.FindPlugins("provider", dirs)
+	for name, provider := range config.Providers {
-	for k, _ := range localPlugins {
+		var fqn addrs.Provider
-		c.ui.Info(fmt.Sprintf("plugin: %s (%s)", k.Name, k.Version))
+		var diags tfdiags.Diagnostics
+		if provider.Source != "" {
+			fqn, diags = addrs.ParseProviderSourceString(provider.Source)
+			if diags.HasErrors() {
+				c.ui.Error(fmt.Sprintf("Invalid provider source string: %s", provider.Source))
+				return 1
+			}
+		} else {
+			fqn = addrs.NewDefaultProvider(name)
+		}
+		reqs[fqn] = provider.Versions
 	}
-	installer := &discovery.ProviderInstaller{
-		Dir: workDir,
-
-		// FIXME: This is incorrect because it uses the protocol version of
-		// this tool, rather than of the Terraform binary we just downloaded.
-		// But we can't get this information from a Terraform binary, so
-		// we'll just ignore this for now and use the same plugin installer
-		// protocol version for terraform-bundle as the terraform shipped
-		// with this release.
-		//
-		// NOTE: To target older versions of terraform, use the terraform-bundle
-		// from the same tag.
-		PluginProtocolVersion: discovery.PluginInstallProtocolVersion,
 
+	// set up the provider installer
+	platform := getproviders.Platform{
 		OS:   osName,
 		Arch: archName,
-		Ui:   c.ui,
 	}
+	installdir := providercache.NewDirWithPlatform(filepath.Join(workDir, "plugins"), platform)
 
-	for name, constraintStrs := range config.Providers {
+	services := disco.New()
-		for _, constraintStr := range constraintStrs {
+	services.SetUserAgent(httpclient.TerraformUserAgent(version.String()))
-			c.ui.Output(fmt.Sprintf("- Resolving %q provider (%s)...",
+	var sources []getproviders.MultiSourceSelector
-				name, constraintStr))
+
-			foundPlugins := discovery.PluginMetaSet{}
+	// Find any local providers first so we can exclude these from the registry
-			constraint := constraintStr.MustParse()
+	// install. We'll just silently ignore any errors and assume it would fail
-			for plugin, _ := range localPlugins {
+	// real installation later too.
-				if plugin.Name == name && constraint.Allows(plugin.Version.MustParse()) {
+	foundLocally := map[addrs.Provider]struct{}{}
-					foundPlugins.Add(plugin)
+
+	if absPluginDir, err := filepath.Abs(pluginDir); err == nil {
+		c.ui.Info(fmt.Sprintf("Local plugin directory %q found; scanning for provider binaries.", pluginDir))
+		if _, err := os.Stat(absPluginDir); err == nil {
+			localSource := getproviders.NewFilesystemMirrorSource(absPluginDir)
+			if available, err := localSource.AllAvailablePackages(); err == nil {
+				for found := range available {
+					c.ui.Info(fmt.Sprintf("Found provider %q in %q. p", found.String(), pluginDir))
+					foundLocally[found] = struct{}{}
 				}
 			}
-
+			sources = append(sources, getproviders.MultiSourceSelector{
-			if len(foundPlugins) > 0 {
+				Source: localSource,
-				plugin := foundPlugins.Newest()
+			})
-				CopyFile(plugin.Path, workDir+"/terraform-provider-"+plugin.Name+"_v"+plugin.Version.MustParse().String()) //put into temp dir
+			if len(foundLocally) == 0 {
-			} else { //attempt to get from the public registry if not found locally
+				c.ui.Info(fmt.Sprintf("No local providers found in %q.", pluginDir))
-				c.ui.Output(fmt.Sprintf("- Checking for provider plugin on %s...",
-					releaseHost))
-				_, _, err := installer.Get(addrs.NewLegacyProvider(name), constraint)
-				if err != nil {
-					c.ui.Error(fmt.Sprintf("- Failed to resolve %s provider %s: %s", name, constraint, err))
-					return 1
-				}
 			}
+		} else {
+			c.ui.Info(fmt.Sprintf("No %q directory found, skipping local provider discovery.", pluginDir))
 		}
 	}
 
-	files, err := ioutil.ReadDir(workDir)
+	// Anything we found in local directories above is excluded from being
+	// looked up via the registry source we're about to construct.
+	var directExcluded getproviders.MultiSourceMatchingPatterns
+	for addr := range foundLocally {
+		directExcluded = append(directExcluded, addr)
+	}
+
+	// Add the registry source, minus any providers found in the local pluginDir.
+	sources = append(sources, getproviders.MultiSourceSelector{
+		Source:  getproviders.NewMemoizeSource(getproviders.NewRegistrySource(services)),
+		Exclude: directExcluded,
+	})
+
+	installer := providercache.NewInstaller(installdir, getproviders.MultiSource(sources))
+
+	err = c.ensureProviderVersions(installer, reqs)
 	if err != nil {
-		c.ui.Error(fmt.Sprintf("Failed to read work directory %s: %s", workDir, err))
+		c.ui.Error(err.Error())
 		return 1
 	}
 
+	// remove the selections.json file created by the provider installer
+	os.Remove(filepath.Join(workDir, "plugins", "selections.json"))
+
 	// If we get this far then our workDir now contains the union of the
 	// contents of all the zip files we downloaded above. We can now create
 	// our output file.
@@ -221,41 +188,99 @@ func (c *PackageCommand) Run(args []string) int {
 		}
 	}()
 
-	for _, file := range files {
+	// recursively walk the workDir to get a list of all binary filepaths
-		if file.IsDir() {
+	err = filepath.Walk(workDir,
-			// should never happen unless something tampers with our tmpdir
+		func(path string, info os.FileInfo, err error) error {
-			continue
+			if err != nil {
-		}
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
 
-		fn := filepath.Join(workDir, file.Name())
+			// maybe symlinks
-		r, err := os.Open(fn)
+			linkPath, err := filepath.EvalSymlinks(path)
-		if err != nil {
+			if err != nil {
-			c.ui.Error(fmt.Sprintf("Failed to open %s: %s", fn, err))
+				return err
-			return 1
+			}
-		}
+			linkInfo, err := os.Stat(linkPath)
-		hdr, err := zip.FileInfoHeader(file)
+			if err != nil {
-		if err != nil {
+				return err
-			c.ui.Error(fmt.Sprintf("Failed to add zip entry for %s: %s", fn, err))
+			}
-			return 1
+
-		}
+			if linkInfo.IsDir() {
-		hdr.Method = zip.Deflate // be sure to compress files
+				// The only time we should encounter a symlink directory is when we
-		w, err := outZ.CreateHeader(hdr)
+				// have a locally-installed provider, so we will grab the provider
-		if err != nil {
+				// binary from that file.
-			c.ui.Error(fmt.Sprintf("Failed to add zip entry for %s: %s", fn, err))
+				files, err := ioutil.ReadDir(linkPath)
-			return 1
+				if err != nil {
-		}
+					return err
-		_, err = io.Copy(w, r)
+				}
-		if err != nil {
+				for _, file := range files {
-			c.ui.Error(fmt.Sprintf("Failed to write %s to bundle: %s", fn, err))
+					if strings.Contains(file.Name(), "terraform-provider") {
-			return 1
+						relPath, _ := filepath.Rel(workDir, path)
-		}
+						return addZipFile(
+							filepath.Join(linkPath, file.Name()), // the link to this provider binary
+							filepath.Join(relPath, file.Name()),  // the expected directory for the binary
+							info, outZ,
+						)
+					}
+				}
+				// This shouldn't happen - we should always find a provider
+				// binary and exit the loop - but on the chance it does not,
+				// just continue.
+				return nil
+			}
+
+			// provider plugins need to be created in the same relative directory structure
+			absPath, err := filepath.Abs(linkPath)
+			if err != nil {
+				return err
+			}
+			relPath, err := filepath.Rel(workDir, absPath)
+			if err != nil {
+				return err
+			}
+
+			return addZipFile(path, relPath, info, outZ)
+
+		})
+
+	if err != nil {
+		c.ui.Error(err.Error())
+		return 1
 	}
-
 	c.ui.Info("All done!")
 
 	return 0
 }
 
+// addZipFile is a helper function intneded to simplify customizing the file
+// path when adding a file to the zip archive. The relPath is specified for
+// provider binaries, which need to be zipped into the full directory hierarchy.
+func addZipFile(fn, relPath string, info os.FileInfo, outZ *zip.Writer) error {
+	hdr, err := zip.FileInfoHeader(info)
+	if err != nil {
+		return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err)
+	}
+	hdr.Method = zip.Deflate // be sure to compress files
+	hdr.Name = relPath       // we need the full, relative path to the provider binary
+	w, err := outZ.CreateHeader(hdr)
+	if err != nil {
+		return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err)
+	}
+
+	r, err := os.Open(fn)
+	if err != nil {
+		return fmt.Errorf("Failed to open %s: %s", fn, err)
+	}
+	_, err = io.Copy(w, r)
+	if err != nil {
+		return fmt.Errorf("Failed to write %s to bundle: %s", fn, err)
+	}
+	return nil
+}
+
 func (c *PackageCommand) bundleFilename(version discovery.VersionStr, time time.Time, osName, archName string) string {
 	time = time.UTC()
 	return fmt.Sprintf(
@@ -306,26 +331,82 @@ not a normal Terraform configuration file. The file format looks like this:
   terraform {
     # Version of Terraform to include in the bundle. An exact version number
 	# is required.
-    version = "0.10.0"
+    version = "0.13.0"
   }
 
   # Define which provider plugins are to be included
   providers {
     # Include the newest "aws" provider version in the 1.0 series.
-    aws = ["~> 1.0"]
+    aws = {
+		versions = ["~> 1.0"]
+	}
 
     # Include both the newest 1.0 and 2.0 versions of the "google" provider.
     # Each item in these lists allows a distinct version to be added. If the
 	# two expressions match different versions then _both_ are included in
 	# the bundle archive.
-	google = ["~> 1.0", "~> 2.0"]
+	google = {
-	
+		versions = ["~> 1.0", "~> 2.0"]
-	#Include a custom plugin to the bundle. Will search for the plugin in the 
+	}
-	#plugins directory, and package it with the bundle archive. Plugin must have
+
-	#a name of the form: terraform-provider-*-v*, and must be built with the operating
+	# Include a custom plugin to the bundle. Will search for the plugin in the 
-	#system and architecture that terraform enterprise is running, e.g. linux and amd64
+	# plugins directory, and package it with the bundle archive. Plugin must 
-	customplugin = ["0.1"]
+	# have a name of the form: terraform-provider-*, and must be built with
+	# the operating system and architecture that terraform enterprise is running,
+	# e.g. linux and amd64.
+	# See the README for more information on the source attribute and plugin
+	# directory layout.
+	customplugin = {
+		versions = ["0.1"]
+		source = "example.com/myorg/customplugin"
+	}
   }
 
 `
 }
+
+// ensureProviderVersions is a wrapper around
+// providercache.EnsureProviderVersions which allows installing multiple
+// versions of a given provider.
+func (c *PackageCommand) ensureProviderVersions(installer *providercache.Installer, reqs map[addrs.Provider][]string) error {
+	mode := providercache.InstallNewProvidersOnly
+	evts := &providercache.InstallerEvents{
+		ProviderAlreadyInstalled: func(provider addrs.Provider, selectedVersion getproviders.Version) {
+			c.ui.Info(fmt.Sprintf("- Using previously-installed %s v%s", provider.ForDisplay(), selectedVersion))
+		},
+		QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints) {
+			if len(versionConstraints) > 0 {
+				c.ui.Info(fmt.Sprintf("- Finding %s versions matching %q...", provider.ForDisplay(), getproviders.VersionConstraintsString(versionConstraints)))
+			} else {
+				c.ui.Info(fmt.Sprintf("- Finding latest version of %s...", provider.ForDisplay()))
+			}
+		},
+		FetchPackageBegin: func(provider addrs.Provider, version getproviders.Version, location getproviders.PackageLocation) {
+			c.ui.Info(fmt.Sprintf("- Installing %s v%s...", provider.ForDisplay(), version))
+		},
+		QueryPackagesFailure: func(provider addrs.Provider, err error) {
+			c.ui.Error(fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s.", provider.ForDisplay(), err))
+		},
+		FetchPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) {
+			c.ui.Error(fmt.Sprintf("Error while installing %s v%s: %s.", provider.ForDisplay(), version, err))
+		},
+	}
+
+	ctx := evts.OnContext(context.TODO())
+	for provider, versions := range reqs {
+		for _, constraint := range versions {
+			req := make(getproviders.Requirements, 1)
+			cstr, err := getproviders.ParseVersionConstraints(constraint)
+			if err != nil {
+				return err
+			}
+			req[provider] = cstr
+			_, err = installer.EnsureProviderVersions(ctx, req, mode)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}