From 5a48530f47d4810b9a5de8d8608679d95ba042f5 Mon Sep 17 00:00:00 2001 From: Kristin Laemmert Date: Thu, 3 Jun 2021 14:08:04 -0400 Subject: [PATCH] tools: remove terraform-bundle. (#28876) * tools: remove terraform-bundle. terraform-bundle is no longer supported in the main branch of terraform. Users can build terraform-bundle from terraform tagged v0.15 and older. * add a README pointing users to the v0.15 branch --- .circleci/config.yml | 2 +- internal/providercache/dir.go | 2 +- tools/terraform-bundle/CHANGELOG.md | 11 - tools/terraform-bundle/README.md | 216 +-------- tools/terraform-bundle/config.go | 87 ---- tools/terraform-bundle/e2etest/doc.go | 2 - tools/terraform-bundle/e2etest/main_test.go | 39 -- .../terraform-bundle/e2etest/package_test.go | 229 ---------- .../testdata/empty/terraform-bundle.hcl | 3 - .../darwin_amd64/terraform-provider-mycloud | 1 - .../local-providers/terraform-bundle.hcl | 11 - .../many-providers/terraform-bundle.hcl | 17 - tools/terraform-bundle/main.go | 81 ---- tools/terraform-bundle/package.go | 423 ------------------ 14 files changed, 6 insertions(+), 1118 deletions(-) delete mode 100644 tools/terraform-bundle/CHANGELOG.md delete mode 100644 tools/terraform-bundle/config.go delete mode 100644 tools/terraform-bundle/e2etest/doc.go delete mode 100644 tools/terraform-bundle/e2etest/main_test.go delete mode 100644 tools/terraform-bundle/e2etest/package_test.go delete mode 100644 tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl delete mode 100644 tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud delete mode 100644 tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl delete mode 100644 tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl delete mode 100644 tools/terraform-bundle/main.go delete mode 100644 tools/terraform-bundle/package.go diff --git a/.circleci/config.yml b/.circleci/config.yml index 943b93a76..20ac37e82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,7 @@ jobs: - run: name: Run Go E2E Tests command: | - gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./internal/command/e2etest ./tools/terraform-bundle/e2etest + gotestsum --format=short-verbose --junitfile $TEST_RESULTS_DIR/gotestsum-report.xml -- -p 2 -cover -coverprofile=cov_e2e.part ./internal/command/e2etest # save coverage report parts - persist_to_workspace: diff --git a/internal/providercache/dir.go b/internal/providercache/dir.go index c33114f86..f58184aa2 100644 --- a/internal/providercache/dir.go +++ b/internal/providercache/dir.go @@ -58,7 +58,7 @@ func NewDir(baseDir string) *Dir { // running. // // This is primarily intended for portable unit testing and not particularly -// useful in "real" callers, with the exception of terraform-bundle. +// useful in "real" callers. func NewDirWithPlatform(baseDir string, platform getproviders.Platform) *Dir { return &Dir{ baseDir: baseDir, diff --git a/tools/terraform-bundle/CHANGELOG.md b/tools/terraform-bundle/CHANGELOG.md deleted file mode 100644 index 80d512045..000000000 --- a/tools/terraform-bundle/CHANGELOG.md +++ /dev/null @@ -1,11 +0,0 @@ -## 0.14.0 - -BUG FIXES: -* fix packaging for custom plugins ([#26394](https://github.com/hashicorp/terraform/pull/26394)) - -## 0.13.0 (August 10, 2020) - -> This is a list of changes relative to terraform-bundle tagged v0.12. - -Breaking Changes: -* Terraform v0.13.0 has introduced a new hierarchical namespace for providers. Terraform v0.13 requires a new directory layout in order to discover locally-installed provider plugins, and terraform-bundle has been updated to match. Please see the [README](README.md) to learn more about the new directory layout. diff --git a/tools/terraform-bundle/README.md b/tools/terraform-bundle/README.md index dc3879627..e95e642f1 100644 --- a/tools/terraform-bundle/README.md +++ b/tools/terraform-bundle/README.md @@ -1,214 +1,6 @@ # terraform-bundle -`terraform-bundle` is a helper program to create "bundle archives", which are -zip files that contain both a particular version of Terraform and a number -of provider plugins. - -Normally `terraform init` will download and install the plugins necessary to -work with a particular configuration, but sometimes Terraform is deployed in -a network that, for one reason or another, cannot access the official -plugin repository for automatic download. - -In some cases, this can be solved by installing provider plugins into the -[user plugins directory](https://www.terraform.io/docs/configuration/providers.html#third-party-plugins). -However, this doesn't always meet the needs of automated deployments. - -`terraform-bundle` provides an alternative, by allowing the auto-download -process to be run out-of-band on a separate machine that _does_ have access -to the repository. The result is a zip file that can be extracted onto the -target system to install both the desired Terraform version and a selection -of providers, thus avoiding the need for on-the-fly plugin installation. - -## Building - -To build `terraform-bundle` from source, set up a Terraform development -environment per [Terraform's own README](../../README.md) and then install -this tool from within it: - -``` -$ go install ./tools/terraform-bundle -``` - -This will install `terraform-bundle` in `$GOPATH/bin`, which is assumed by -the rest of this README to be in `PATH`. - -`terraform-bundle` is a repackaging of the module installation functionality -from Terraform itself, so for best results you should build from the tag -relating to the version of Terraform you plan to use. For example, use the v0.12 -tag to build a version of terraform-bundle compatible with Terraform v0.12*. - -## Usage - -`terraform-bundle` uses a simple configuration file to define what should -be included in a bundle. This is designed so that it can be checked into -version control and used by an automated build and deploy process. - -The configuration file format works as follows: - -```hcl -terraform { - # Version of Terraform to include in the bundle. An exact version number - # is required. - version = "0.10.0" -} - -# Define which provider plugins are to be included -providers { - # Include the newest "aws" provider version in the 1.0 series. - aws = { - versions = ["~> 1.0"] - } - - # Include both the newest 1.0 and 2.0 versions of the "google" provider. - # Each item in these lists allows a distinct version to be added. If the - # two expressions match different versions then _both_ are included in - # the bundle archive. - google = { - versions = ["~> 1.0", "~> 2.0"] - } - - # Include a custom plugin to the bundle. Will search for the plugin in the - # plugins directory and package it with the bundle archive. Plugin must have - # a name of the form: terraform-provider-*, and must be built with the operating - # system and architecture that terraform enterprise is running, e.g. linux and amd64. - customplugin = { - versions = ["0.1"] - source = "myorg/customplugin" - } -} - -``` - -The `terraform` block defines which version of Terraform will be included -in the bundle. An exact version is required here. - -The `providers` block defines zero or more providers to include in the bundle -along with core Terraform. Each attribute is a provider name, and its value is a -block with the list of version constraints and (optional) source. For each given -constraint, `terraform-bundle` will find the newest available version matching -the constraint and include it in the bundle. - -It is allowed to specify multiple constraints for the same provider, in which -case multiple versions can be included in the resulting bundle. Each constraint -string given results in a separate plugin in the bundle, unless two constraints -resolve to the same concrete plugin. - -Including multiple versions of the same provider allows several configurations -running on the same system to share an installation of the bundle and to -choose a version using version constraints within the main Terraform -configuration. This avoids the need to upgrade all configurations to newer -versions in lockstep. - -After creating the configuration file, e.g. `terraform-bundle.hcl`, a bundle -zip file can be produced as follows: - -``` -$ terraform-bundle package terraform-bundle.hcl -``` - -By default the bundle package will target the operating system and CPU -architecture where the tool is being run. To override this, use the `-os` and -`-arch` options. For example, to build a bundle for on-premises Terraform -Enterprise: - -``` -$ terraform-bundle package -os=linux -arch=amd64 terraform-bundle.hcl -``` - -The bundle file is assigned a name that includes the core Terraform version -number, a timestamp to the nearest hour of when the bundle was built, and the -target OS and CPU architecture. It is recommended to refer to a bundle using -this composite version number so that bundle archives can be easily -distinguished from official release archives and from each other when multiple -bundles contain the same core Terraform version. - -## Custom Plugins -To include custom plugins in the bundle file, create a local directory named -`./.plugins` and put all the plugins you want to include there, under the -required [sub directory](#plugins-directory-layout). Optionally, you can use the -`-plugin-dir` flag to specify a location where to find the plugins. To be -recognized as a valid plugin, the file must have a name of the form -`terraform-provider-`. In addition, ensure that the plugin is built using -the same operating system and architecture used for Terraform Enterprise. -Typically this will be `linux` and `amd64`. - -### Plugins Directory Layout -To include custom plugins in the bundle file, you must specify a "source" -attribute in the configuration and place the plugin in the appropriate -subdirectory under `./.plugins`. The directory must have the following layout: - -``` -./.plugins/$SOURCEHOST/$SOURCENAMESPACE/$NAME/$VERSION/$OS_$ARCH/ -``` - -When installing custom plugins, you may choose any arbitrary identifier for the -$SOURCEHOST and $SOURCENAMESPACE subdirectories. - -For example, given the following configuration and a plugin built for Terraform Enterprise: - -``` -providers { - customplugin = { - versions = ["0.1"] - source = "example.com/myorg/customplugin" - } -} -``` - -The binary must be placed in the following directory: - -``` -./.plugins/example.com/myorg/customplugin/0.1/linux_amd64/ -``` - -## Provider Resolution Behavior - -Terraform's provider resolution behavior is such that if a given constraint -can be resolved by any plugin already installed on the system it will use -the newest matching plugin and not attempt automatic installation. - -Therefore if automatic installation is not desired, it is important to ensure -that version constraints within Terraform configurations do not exclude all -of the versions available from the bundle. If a suitable version cannot be -found in the bundle, Terraform _will_ attempt to satisfy that dependency by -automatic installation from the official repository. - -For full details about provider resolution, see -[How Terraform Works: Plugin Discovery](https://www.terraform.io/docs/extend/how-terraform-works.html#discovery). - -The downloaded provider archives are verified using the same signature check -that is used for auto-installed plugins, using Hashicorp's release key. At -this time, the core Terraform archive itself is _not_ verified in this way; -that may change in a future version of this tool. - -## Installing a Bundle in Terraform Enterprise - -If using a Terraform Enterprise instance in an "air-gapped" -environment, this tool can produce a custom Terraform version package, which -includes a set of provider plugins along with core Terraform. - -To create a suitable bundle, use the `-os` and `-arch` options as described -above to produce a bundle targeting `linux_amd64`. You can then place this -archive on an HTTP server reachable by the Terraform Enterprise hosts and -install it as per -[Administration: Managing Terraform Versions](https://www.terraform.io/docs/enterprise/admin/resources.html#managing-terraform-versions). - -After clicking the "Add Terraform Version" button: - -1. In the "Version" field, enter the generated bundle version from the bundle - filename, which will be of the form `N.N.N-bundleYYYYMMDDHH`. -2. In the "URL" field, enter the URL where the generated bundle archive can be found. -3. In the "SHA256 Checksum" field, enter the SHA256 hash of the file, which can - be found by running `sha256sum ` or `shasum -a256 `. - -The new bundle version can then be selected as the Terraform version for -any workspace. When selected, configurations that require only plugins -included in the bundle will run without trying to auto-install. - -Note that the above does _not_ apply to Terraform Pro, or to Terraform Premium -when not running a private install. In these packages, Terraform versions -are managed centrally across _all_ organizations and so custom bundles are not -supported. - -For more information on the available Terraform Enterprise packages, see -[the Terraform product site](https://www.hashicorp.com/products/terraform/). +terraform-bundle is no longer actively maintained. We recommend that you switch +to one of the [alternative provider installation methods](https://www.terraform.io/docs/cli/config/config-file.html#provider-installation) +introduced in Terraform v0.13. To continue using terraform-bundle, you can build +terraform-bundle from the v0.15 branch of the terraform repository. diff --git a/tools/terraform-bundle/config.go b/tools/terraform-bundle/config.go deleted file mode 100644 index 74bcad96c..000000000 --- a/tools/terraform-bundle/config.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - - "github.com/hashicorp/hcl" - "github.com/hashicorp/terraform/internal/addrs" - "github.com/hashicorp/terraform/internal/getproviders" - "github.com/hashicorp/terraform/internal/plugin/discovery" -) - -var zeroThirteen = discovery.ConstraintStr(">= 0.13.0").MustParse() - -type Config struct { - Terraform TerraformConfig `hcl:"terraform"` - Providers map[string]ProviderConfig `hcl:"providers"` -} - -type TerraformConfig struct { - Version discovery.VersionStr `hcl:"version"` -} - -type ProviderConfig struct { - Versions []string `hcl:"versions"` - Source string `hcl:"source"` -} - -func LoadConfig(src []byte, filename string) (*Config, error) { - config := &Config{} - err := hcl.Decode(config, string(src)) - if err != nil { - return config, err - } - - err = config.validate() - return config, err -} - -func LoadConfigFile(filename string) (*Config, error) { - src, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - - return LoadConfig(src, filename) -} - -func (c *Config) validate() error { - if c.Terraform.Version == "" { - return fmt.Errorf("terraform.version is required") - } - - var v discovery.Version - var err error - if v, err = c.Terraform.Version.Parse(); err != nil { - return fmt.Errorf("terraform.version: %s", err) - } - - if !zeroThirteen.Allows(v) { - return fmt.Errorf("this version of terraform-bundle can only build bundles for Terraform v0.13 and later; build terraform-bundle from a release tag (such as v0.12.*) to construct bundles for earlier versions") - } - - if c.Providers == nil { - c.Providers = map[string]ProviderConfig{} - } - - for k, cs := range c.Providers { - if cs.Source != "" { - _, diags := addrs.ParseProviderSourceString(cs.Source) - if diags.HasErrors() { - return fmt.Errorf("providers.%s: %s", k, diags.Err().Error()) - } - } - if len(cs.Versions) > 0 { - for _, c := range cs.Versions { - if _, err := getproviders.ParseVersionConstraints(c); err != nil { - return fmt.Errorf("providers.%s: %s", k, err) - } - } - } else { - return fmt.Errorf("provider.%s: required \"versions\" argument not found", k) - } - } - - return nil -} diff --git a/tools/terraform-bundle/e2etest/doc.go b/tools/terraform-bundle/e2etest/doc.go deleted file mode 100644 index bc2dea42f..000000000 --- a/tools/terraform-bundle/e2etest/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// terraform bundle e2e tests -package e2etest diff --git a/tools/terraform-bundle/e2etest/main_test.go b/tools/terraform-bundle/e2etest/main_test.go deleted file mode 100644 index f59c670d3..000000000 --- a/tools/terraform-bundle/e2etest/main_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package e2etest - -import ( - "os" - "testing" - - "github.com/hashicorp/terraform/internal/e2e" -) - -var bundleBin string - -func TestMain(m *testing.M) { - teardown := setup() - code := m.Run() - teardown() - os.Exit(code) -} - -func setup() func() { - tmpFilename := e2e.GoBuild("github.com/hashicorp/terraform/tools/terraform-bundle", "terraform-bundle") - bundleBin = tmpFilename - - return func() { - os.Remove(tmpFilename) - } -} - -func canAccessNetwork() bool { - // We re-use the flag normally used for acceptance tests since that's - // established as a way to opt-in to reaching out to real systems that - // may suffer transient errors. - return os.Getenv("TF_ACC") != "" -} - -func skipIfCannotAccessNetwork(t *testing.T) { - if !canAccessNetwork() { - t.Skip("network access not allowed; use TF_ACC=1 to enable") - } -} diff --git a/tools/terraform-bundle/e2etest/package_test.go b/tools/terraform-bundle/e2etest/package_test.go deleted file mode 100644 index 74332fa32..000000000 --- a/tools/terraform-bundle/e2etest/package_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package e2etest - -import ( - "archive/zip" - "fmt" - "io/ioutil" - "path/filepath" - "runtime" - "strings" - "testing" - - "github.com/hashicorp/terraform/internal/e2e" -) - -func TestPackage_empty(t *testing.T) { - t.Parallel() - // This test reaches out to releases.hashicorp.com to download the - // template provider, so it can only run if network access is allowed. - // We intentionally don't try to stub this here, because there's already - // a stubbed version of this in the "command" package and so the goal here - // is to test the interaction with the real repository. - skipIfCannotAccessNetwork(t) - - fixturePath := filepath.Join("testdata", "empty") - tfBundle := e2e.NewBinary(bundleBin, fixturePath) - defer tfBundle.Close() - - stdout, stderr, err := tfBundle.Run("package", "terraform-bundle.hcl") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - if stderr != "" { - t.Errorf("unexpected stderr output:\n%s", stderr) - } - - if !strings.Contains(stdout, "Fetching Terraform 0.13.0 core package...") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "Creating terraform_0.13.0-bundle") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "All done!") { - t.Errorf("success message is missing from output:\n%s", stdout) - } -} - -func TestPackage_manyProviders(t *testing.T) { - t.Parallel() - - // This test reaches out to releases.hashicorp.com to download providers, so - // it can only run if network access is allowed. We intentionally don't try - // to stub this here, because there's already a stubbed version of this in - // the "command" package and so the goal here is to test the interaction - // with the real repository. - skipIfCannotAccessNetwork(t) - - fixturePath := filepath.Join("testdata", "many-providers") - tfBundle := e2e.NewBinary(bundleBin, fixturePath) - defer tfBundle.Close() - - stdout, stderr, err := tfBundle.Run("package", "terraform-bundle.hcl") - if err != nil { - t.Errorf("unexpected error: %s", err) - } - - if stderr != "" { - t.Errorf("unexpected stderr output:\n%s", stderr) - } - - // Here we have to check each provider separately - // because it's internally held in a map (i.e. not guaranteed order) - - if !strings.Contains(stdout, `- Finding hashicorp/aws versions matching "~> 2.26.0"... -- Installing hashicorp/aws v2.26.0...`) { - t.Errorf("success message is missing from output:\n%s", stdout) - } - - if !strings.Contains(stdout, `- Finding hashicorp/kubernetes versions matching "1.8.0"... -- Installing hashicorp/kubernetes v1.8.0... -- Finding hashicorp/kubernetes versions matching "1.8.1"... -- Installing hashicorp/kubernetes v1.8.1... -- Finding hashicorp/kubernetes versions matching "1.9.0"... -- Installing hashicorp/kubernetes v1.9.0...`) { - t.Errorf("success message is missing from output:\n%s", stdout) - } - - if !strings.Contains(stdout, `- Finding hashicorp/null versions matching "2.1.0"... -- Installing hashicorp/null v2.1.0...`) { - t.Errorf("success message is missing from output:\n%s", stdout) - } - - if !strings.Contains(stdout, "Fetching Terraform 0.13.0 core package...") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "Creating terraform_0.13.0-bundle") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "All done!") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - - // check the contents of the created zipfile - files, err := ioutil.ReadDir(tfBundle.WorkDir()) - if err != nil { - t.Fatalf("error reading workdir: %s", err) - } - for _, file := range files { - if strings.Contains(file.Name(), "terraform_0.13.0-bundle") { - read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name())) - if err != nil { - t.Fatalf("Failed to open archive: %s", err) - } - defer read.Close() - - expectedFiles := map[string]struct{}{ - "terraform": {}, - testProviderBinaryPath("null", "2.1.0"): {}, - testProviderBinaryPath("aws", "2.26.0"): {}, - testProviderBinaryPath("kubernetes", "1.8.0"): {}, - testProviderBinaryPath("kubernetes", "1.8.1"): {}, - testProviderBinaryPath("kubernetes", "1.9.0"): {}, - } - extraFiles := make(map[string]struct{}) - - for _, file := range read.File { - if _, exists := expectedFiles[file.Name]; exists { - if !file.FileInfo().Mode().IsRegular() { - t.Errorf("Expected file is not a regular file: %s", file.Name) - } - delete(expectedFiles, file.Name) - } else { - extraFiles[file.Name] = struct{}{} - } - } - if len(expectedFiles) != 0 { - t.Errorf("missing expected file(s): %#v", expectedFiles) - } - if len(extraFiles) != 0 { - t.Errorf("found extra unexpected file(s): %#v", extraFiles) - } - } - } -} - -func TestPackage_localProviders(t *testing.T) { - t.Parallel() - - // This test reaches out to releases.hashicorp.com to download terrafrom, so - // it can only run if network access is allowed. The providers are installed - // from the local cache. - skipIfCannotAccessNetwork(t) - - fixturePath := filepath.Join("testdata", "local-providers") - tfBundle := e2e.NewBinary(bundleBin, fixturePath) - defer tfBundle.Close() - - // we explicitly specify the platform so that tests can find the local binary under the expected directory - stdout, stderr, err := tfBundle.Run("package", "-os=darwin", "-arch=amd64", "terraform-bundle.hcl") - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - - if stderr != "" { - t.Errorf("unexpected stderr output:\n%s", stderr) - } - - // Here we have to check each provider separately - // because it's internally held in a map (i.e. not guaranteed order) - if !strings.Contains(stdout, "Fetching Terraform 0.13.0 core package...") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "Creating terraform_0.13.0-bundle") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - if !strings.Contains(stdout, "All done!") { - t.Errorf("success message is missing from output:\n%s", stdout) - } - - // check the contents of the created zipfile - files, err := ioutil.ReadDir(tfBundle.WorkDir()) - if err != nil { - t.Fatalf("error reading workdir: %s", err) - } - for _, file := range files { - if strings.Contains(file.Name(), "terraform_0.13.0-bundle") { - read, err := zip.OpenReader(filepath.Join(tfBundle.WorkDir(), file.Name())) - if err != nil { - t.Fatalf("Failed to open archive: %s", err) - } - defer read.Close() - - expectedFiles := map[string]struct{}{ - "terraform": {}, - "plugins/example.com/myorg/mycloud/0.1.0/darwin_amd64/terraform-provider-mycloud": {}, - } - extraFiles := make(map[string]struct{}) - - for _, file := range read.File { - if _, exists := expectedFiles[file.Name]; exists { - if !file.FileInfo().Mode().IsRegular() { - t.Errorf("Expected file is not a regular file: %s", file.Name) - } - delete(expectedFiles, file.Name) - } else { - extraFiles[file.Name] = struct{}{} - } - } - if len(expectedFiles) != 0 { - t.Errorf("missing expected file(s): %#v", expectedFiles) - } - if len(extraFiles) != 0 { - t.Errorf("found extra unexpected file(s): %#v", extraFiles) - } - } - } -} - -// testProviderBinaryPath takes a provider name (assumed to be a hashicorp -// provider) and version and returns the expected binary path, relative to the -// archive, for the plugin. -func testProviderBinaryPath(provider, version string) string { - os := runtime.GOOS - arch := runtime.GOARCH - return fmt.Sprintf( - "plugins/registry.terraform.io/hashicorp/%s/%s/%s_%s/terraform-provider-%s_v%s_x4", - provider, version, os, arch, provider, version, - ) -} diff --git a/tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl b/tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl deleted file mode 100644 index 5a0302091..000000000 --- a/tools/terraform-bundle/e2etest/testdata/empty/terraform-bundle.hcl +++ /dev/null @@ -1,3 +0,0 @@ -terraform { - version = "0.13.0" -} diff --git a/tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud b/tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud deleted file mode 100644 index bd10a3eda..000000000 --- a/tools/terraform-bundle/e2etest/testdata/local-providers/.plugins/example.com/myorg/mycloud/0.1/darwin_amd64/terraform-provider-mycloud +++ /dev/null @@ -1 +0,0 @@ -I am a fake binary. diff --git a/tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl b/tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl deleted file mode 100644 index fd55f6777..000000000 --- a/tools/terraform-bundle/e2etest/testdata/local-providers/terraform-bundle.hcl +++ /dev/null @@ -1,11 +0,0 @@ -terraform { - version = "0.13.0" -} - -providers { - // this provider is installed in .plugins - mycloud = { - versions = ["0.1"] - source = "example.com/myorg/mycloud" - } -} diff --git a/tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl b/tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl deleted file mode 100644 index 93af2b95b..000000000 --- a/tools/terraform-bundle/e2etest/testdata/many-providers/terraform-bundle.hcl +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - version = "0.13.0" -} - -providers { - aws = { - versions = ["~> 2.26.0"] - } - - kubernetes = { - versions = ["1.8.0", "1.8.1", "1.9.0"] - } - - null = { - versions = ["2.1.0"] - } -} diff --git a/tools/terraform-bundle/main.go b/tools/terraform-bundle/main.go deleted file mode 100644 index 1e1ba08f8..000000000 --- a/tools/terraform-bundle/main.go +++ /dev/null @@ -1,81 +0,0 @@ -// terraform-bundle is a tool to create "bundle archives" that contain both -// a particular version of Terraform and a set of providers for use with it. -// -// Such bundles are useful for distributing a Terraform version and a set -// of providers to a system out-of-band, in situations where Terraform's -// auto-installer cannot be used due to firewall rules, "air-gapped" systems, -// etc. -// -// When using bundle archives, it's suggested to use a version numbering -// scheme that adds a suffix that identifies the archive as being a bundle, -// to make it easier to distinguish bundle archives from the normal separated -// release archives. This tool by default produces files with the following -// naming scheme: -// -// terraform_0.10.0-bundle2017070302_linux_amd64.zip -// -// The user is free to rename these files, since the archive filename has -// no significance to Terraform itself and the generated pseudo-version number -// is not referenced within the archive contents. -// -// If using such a bundle with an on-premises Terraform Enterprise installation, -// it's recommended to use the generated version number (or a modification -// thereof) as the tool version within Terraform Enterprise, so that -// bundle archives can be distinguished from official releases and from -// each other even if the same core Terraform version is used. -// -// Terraform providers in general release more often than core, so it is -// intended that this tool can be used to periodically upgrade providers -// within certain constraints and produce a new bundle containing these -// upgraded provider versions. A bundle archive can include multiple versions -// of the same provider, allowing configurations containing provider version -// constrants to be gradually migrated to newer versions. -package main - -import ( - "io/ioutil" - "log" - "os" - - tfversion "github.com/hashicorp/terraform/version" - "github.com/mitchellh/cli" -) - -func main() { - ui := &cli.ColoredUi{ - OutputColor: cli.UiColorNone, - InfoColor: cli.UiColorNone, - ErrorColor: cli.UiColorRed, - WarnColor: cli.UiColorYellow, - - Ui: &cli.BasicUi{ - Reader: os.Stdin, - Writer: os.Stdout, - ErrorWriter: os.Stderr, - }, - } - - // Terraform's code tends to produce noisy logs, since Terraform itself - // suppresses them by default. To avoid polluting our console, we'll do - // the same. - if os.Getenv("TF_LOG") == "" { - log.SetOutput(ioutil.Discard) - } - - c := cli.NewCLI("terraform-bundle", tfversion.Version) - c.Args = os.Args[1:] - c.Commands = map[string]cli.CommandFactory{ - "package": func() (cli.Command, error) { - return &PackageCommand{ - ui: ui, - }, nil - }, - } - - exitStatus, err := c.Run() - if err != nil { - ui.Error(err.Error()) - } - - os.Exit(exitStatus) -} diff --git a/tools/terraform-bundle/package.go b/tools/terraform-bundle/package.go deleted file mode 100644 index d2831c087..000000000 --- a/tools/terraform-bundle/package.go +++ /dev/null @@ -1,423 +0,0 @@ -package main - -import ( - "archive/zip" - "context" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "strings" - "time" - - getter "github.com/hashicorp/go-getter" - "github.com/hashicorp/terraform-svchost/disco" - "github.com/hashicorp/terraform/internal/addrs" - "github.com/hashicorp/terraform/internal/depsfile" - "github.com/hashicorp/terraform/internal/getproviders" - "github.com/hashicorp/terraform/internal/httpclient" - discovery "github.com/hashicorp/terraform/internal/plugin/discovery" - "github.com/hashicorp/terraform/internal/providercache" - "github.com/hashicorp/terraform/internal/tfdiags" - "github.com/hashicorp/terraform/version" - "github.com/mitchellh/cli" -) - -var releaseHost = "https://releases.hashicorp.com" - -var pluginDir = ".plugins" - -type PackageCommand struct { - ui cli.Ui -} - -func (c *PackageCommand) Run(args []string) int { - flags := flag.NewFlagSet("package", flag.ExitOnError) - osPtr := flags.String("os", "", "Target operating system") - archPtr := flags.String("arch", "", "Target CPU architecture") - pluginDirPtr := flags.String("plugin-dir", "", "Path to custom plugins directory") - err := flags.Parse(args) - if err != nil { - c.ui.Error(err.Error()) - return 1 - } - - osName := runtime.GOOS - archName := runtime.GOARCH - if *osPtr != "" { - osName = *osPtr - } - if *archPtr != "" { - archName = *archPtr - } - if *pluginDirPtr != "" { - pluginDir = *pluginDirPtr - } - - if flags.NArg() != 1 { - c.ui.Error("Configuration filename is required") - return 1 - } - configFn := flags.Arg(0) - - config, err := LoadConfigFile(configFn) - if err != nil { - c.ui.Error(fmt.Sprintf("Failed to read config: %s", err)) - return 1 - } - - tmpDir, err := ioutil.TempDir("", "terraform-bundle") - if err != nil { - c.ui.Error(fmt.Sprintf("Could not create temporary dir: %s", err)) - return 1 - } - // symlinked tmp directories can cause odd behaviors. - workDir, err := filepath.EvalSymlinks(tmpDir) - if err != nil { - c.ui.Error(fmt.Sprintf("Error evaulating symlinks: %s", err)) - return 1 - } - defer os.RemoveAll(workDir) - - c.ui.Info(fmt.Sprintf("Fetching Terraform %s core package...", config.Terraform.Version)) - - coreZipURL := c.coreURL(config.Terraform.Version, osName, archName) - err = getter.Get(workDir, coreZipURL) - if err != nil { - c.ui.Error(fmt.Sprintf("Failed to fetch core package from %s: %s", coreZipURL, err)) - return 1 - } - - // get the list of required providers from the config - reqs := make(map[addrs.Provider][]string) - for name, provider := range config.Providers { - var fqn addrs.Provider - var diags tfdiags.Diagnostics - if provider.Source != "" { - fqn, diags = addrs.ParseProviderSourceString(provider.Source) - if diags.HasErrors() { - c.ui.Error(fmt.Sprintf("Invalid provider source string: %s", provider.Source)) - return 1 - } - } else { - fqn = addrs.NewDefaultProvider(name) - } - reqs[fqn] = provider.Versions - } - - // set up the provider installer - platform := getproviders.Platform{ - OS: osName, - Arch: archName, - } - installdir := providercache.NewDirWithPlatform(filepath.Join(workDir, "plugins"), platform) - - services := disco.New() - services.SetUserAgent(httpclient.TerraformUserAgent(version.String())) - var sources []getproviders.MultiSourceSelector - - // Find any local providers first so we can exclude these from the registry - // install. We'll just silently ignore any errors and assume it would fail - // real installation later too. - foundLocally := map[addrs.Provider]struct{}{} - - if absPluginDir, err := filepath.Abs(pluginDir); err == nil { - c.ui.Info(fmt.Sprintf("Local plugin directory %q found; scanning for provider binaries.", pluginDir)) - if _, err := os.Stat(absPluginDir); err == nil { - localSource := getproviders.NewFilesystemMirrorSource(absPluginDir) - if available, err := localSource.AllAvailablePackages(); err == nil { - for found := range available { - c.ui.Info(fmt.Sprintf("Found provider %q in %q.", found.String(), pluginDir)) - foundLocally[found] = struct{}{} - } - } - sources = append(sources, getproviders.MultiSourceSelector{ - Source: localSource, - }) - if len(foundLocally) == 0 { - c.ui.Info(fmt.Sprintf("No local providers found in %q.", pluginDir)) - } - } else { - c.ui.Info(fmt.Sprintf("No %q directory found, skipping local provider discovery.", pluginDir)) - } - } - - // Anything we found in local directories above is excluded from being - // looked up via the registry source we're about to construct. - var directExcluded getproviders.MultiSourceMatchingPatterns - for addr := range foundLocally { - directExcluded = append(directExcluded, addr) - } - - // Add the registry source, minus any providers found in the local pluginDir. - sources = append(sources, getproviders.MultiSourceSelector{ - Source: getproviders.NewMemoizeSource(getproviders.NewRegistrySource(services)), - Exclude: directExcluded, - }) - - installer := providercache.NewInstaller(installdir, getproviders.MultiSource(sources)) - - err = c.ensureProviderVersions(installer, reqs) - if err != nil { - c.ui.Error(err.Error()) - return 1 - } - - // remove the selections.json file created by the provider installer - os.Remove(filepath.Join(workDir, "plugins", "selections.json")) - - // If we get this far then our workDir now contains the union of the - // contents of all the zip files we downloaded above. We can now create - // our output file. - outFn := c.bundleFilename(config.Terraform.Version, time.Now(), osName, archName) - c.ui.Info(fmt.Sprintf("Creating %s ...", outFn)) - outF, err := os.OpenFile(outFn, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - c.ui.Error(fmt.Sprintf("Failed to create %s: %s", outFn, err)) - return 1 - } - outZ := zip.NewWriter(outF) - defer func() { - err := outZ.Close() - if err != nil { - c.ui.Error(fmt.Sprintf("Failed to close %s: %s", outFn, err)) - os.Exit(1) - } - err = outF.Close() - if err != nil { - c.ui.Error(fmt.Sprintf("Failed to close %s: %s", outFn, err)) - os.Exit(1) - } - }() - - // recursively walk the workDir to get a list of all binary filepaths - err = filepath.Walk(workDir, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - // maybe symlinks - linkPath, err := filepath.EvalSymlinks(path) - if err != nil { - return err - } - linkInfo, err := os.Stat(linkPath) - if err != nil { - return err - } - - if linkInfo.IsDir() { - // The only time we should encounter a symlink directory is when we - // have a locally-installed provider, so we will grab the provider - // binary from that file. - files, err := ioutil.ReadDir(linkPath) - if err != nil { - return err - } - for _, file := range files { - if strings.Contains(file.Name(), "terraform-provider") { - relPath, _ := filepath.Rel(workDir, path) - return addZipFile( - filepath.Join(linkPath, file.Name()), // the link to this provider binary - filepath.Join(relPath, file.Name()), // the expected directory for the binary - file, outZ, - ) - } - } - // This shouldn't happen - we should always find a provider - // binary and exit the loop - but on the chance it does not, - // just continue. - return nil - } - - // provider plugins need to be created in the same relative directory structure - absPath, err := filepath.Abs(linkPath) - if err != nil { - return err - } - relPath, err := filepath.Rel(workDir, absPath) - if err != nil { - return err - } - - return addZipFile(path, relPath, info, outZ) - - }) - - if err != nil { - c.ui.Error(err.Error()) - return 1 - } - c.ui.Info("All done!") - - return 0 -} - -// addZipFile is a helper function intneded to simplify customizing the file -// path when adding a file to the zip archive. The relPath is specified for -// provider binaries, which need to be zipped into the full directory hierarchy. -func addZipFile(fn, relPath string, info os.FileInfo, outZ *zip.Writer) error { - hdr, err := zip.FileInfoHeader(info) - if err != nil { - return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err) - } - hdr.Method = zip.Deflate // be sure to compress files - hdr.Name = relPath // we need the full, relative path to the provider binary - w, err := outZ.CreateHeader(hdr) - if err != nil { - return fmt.Errorf("Failed to add zip entry for %s: %s", fn, err) - } - - r, err := os.Open(fn) - if err != nil { - return fmt.Errorf("Failed to open %s: %s", fn, err) - } - _, err = io.Copy(w, r) - if err != nil { - return fmt.Errorf("Failed to write %s to bundle: %s", fn, err) - } - return nil -} - -func (c *PackageCommand) bundleFilename(version discovery.VersionStr, time time.Time, osName, archName string) string { - time = time.UTC() - return fmt.Sprintf( - "terraform_%s-bundle%04d%02d%02d%02d_%s_%s.zip", - version, - time.Year(), time.Month(), time.Day(), time.Hour(), - osName, archName, - ) -} - -func (c *PackageCommand) coreURL(version discovery.VersionStr, osName, archName string) string { - return fmt.Sprintf( - "%s/terraform/%s/terraform_%s_%s_%s.zip", - releaseHost, version, version, osName, archName, - ) -} - -func (c *PackageCommand) Synopsis() string { - return "Produces a bundle archive" -} - -func (c *PackageCommand) Help() string { - return `Usage: terraform-bundle package [options] - -Uses the given bundle configuration file to produce a zip file in the -current working directory containing a Terraform binary along with zero or -more provider plugin binaries. - -Options: - -os=name Target operating system the archive will be built for. Defaults - to that of the system where the command is being run. - - -arch=name Target CPU architecture the archive will be built for. Defaults - to that of the system where the command is being run. - - -plugin-dir=path The path to the custom plugins directory. Defaults to "./plugins". - -The resulting zip file can be used to more easily install Terraform and -a fixed set of providers together on a server, so that Terraform's provider -auto-installation mechanism can be avoided. - -To build an archive for Terraform Enterprise, use: - -os=linux -arch=amd64 - -Note that the given configuration file is a format specific to this command, -not a normal Terraform configuration file. The file format looks like this: - - terraform { - # Version of Terraform to include in the bundle. An exact version number - # is required. - version = "0.13.0" - } - - # Define which provider plugins are to be included - providers { - # Include the newest "aws" provider version in the 1.0 series. - aws = { - versions = ["~> 1.0"] - } - - # Include both the newest 1.0 and 2.0 versions of the "google" provider. - # Each item in these lists allows a distinct version to be added. If the - # two expressions match different versions then _both_ are included in - # the bundle archive. - google = { - versions = ["~> 1.0", "~> 2.0"] - } - - # Include a custom plugin to the bundle. Will search for the plugin in the - # plugins directory, and package it with the bundle archive. Plugin must - # have a name of the form: terraform-provider-*, and must be built with - # the operating system and architecture that terraform enterprise is running, - # e.g. linux and amd64. - # See the README for more information on the source attribute and plugin - # directory layout. - customplugin = { - versions = ["0.1"] - source = "example.com/myorg/customplugin" - } - } - -` -} - -// ensureProviderVersions is a wrapper around -// providercache.EnsureProviderVersions which allows installing multiple -// versions of a given provider. -func (c *PackageCommand) ensureProviderVersions(installer *providercache.Installer, reqs map[addrs.Provider][]string) error { - mode := providercache.InstallNewProvidersOnly - evts := &providercache.InstallerEvents{ - ProviderAlreadyInstalled: func(provider addrs.Provider, selectedVersion getproviders.Version) { - c.ui.Info(fmt.Sprintf("- Using previously-installed %s v%s", provider.ForDisplay(), selectedVersion)) - }, - QueryPackagesBegin: func(provider addrs.Provider, versionConstraints getproviders.VersionConstraints, locked bool) { - if len(versionConstraints) > 0 { - c.ui.Info(fmt.Sprintf("- Finding %s versions matching %q...", provider.ForDisplay(), getproviders.VersionConstraintsString(versionConstraints))) - } else { - c.ui.Info(fmt.Sprintf("- Finding latest version of %s...", provider.ForDisplay())) - } - }, - FetchPackageBegin: func(provider addrs.Provider, version getproviders.Version, location getproviders.PackageLocation) { - c.ui.Info(fmt.Sprintf("- Installing %s v%s...", provider.ForDisplay(), version)) - }, - QueryPackagesFailure: func(provider addrs.Provider, err error) { - c.ui.Error(fmt.Sprintf("Could not retrieve the list of available versions for provider %s: %s.", provider.ForDisplay(), err)) - }, - FetchPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) { - c.ui.Error(fmt.Sprintf("Error while installing %s v%s: %s.", provider.ForDisplay(), version, err)) - }, - } - - ctx := evts.OnContext(context.TODO()) - for provider, versions := range reqs { - for _, constraint := range versions { - req := make(getproviders.Requirements, 1) - cstr, err := getproviders.ParseVersionConstraints(constraint) - if err != nil { - return err - } - req[provider] = cstr - - // We always start with no locks here, because we want to take - // the newest version matching the given version constraint, and - // never consider anything that might've been selected before. - locks := depsfile.NewLocks() - - _, err = installer.EnsureProviderVersions(ctx, locks, req, mode) - if err != nil { - return err - } - } - } - - return nil -}