command/cliconfig: Explicit provider installation method configuration

This set of commits allows explicit configuration of provider installation methods
in the CLI config, overriding the implicit method selections.
This commit is contained in:
Martin Atkins 2020-04-23 10:58:00 -07:00 committed by GitHub
commit 1ce3c60693
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 938 additions and 146 deletions

View File

@ -17,7 +17,7 @@ import (
"github.com/hashicorp/hcl" "github.com/hashicorp/hcl"
"github.com/hashicorp/terraform-svchost" svchost "github.com/hashicorp/terraform-svchost"
"github.com/hashicorp/terraform/tfdiags" "github.com/hashicorp/terraform/tfdiags"
) )
@ -42,6 +42,12 @@ type Config struct {
Credentials map[string]map[string]interface{} `hcl:"credentials"` Credentials map[string]map[string]interface{} `hcl:"credentials"`
CredentialsHelpers map[string]*ConfigCredentialsHelper `hcl:"credentials_helper"` CredentialsHelpers map[string]*ConfigCredentialsHelper `hcl:"credentials_helper"`
// ProviderInstallation represents any provider_installation blocks
// in the configuration. Only one of these is allowed across the whole
// configuration, but we decode into a slice here so that we can handle
// that validation at validation time rather than initial decode time.
ProviderInstallation []*ProviderInstallation
} }
// ConfigHost is the structure of the "host" nested block within the CLI // ConfigHost is the structure of the "host" nested block within the CLI
@ -91,12 +97,23 @@ func LoadConfig() (*Config, tfdiags.Diagnostics) {
} }
} }
if configDir, err := ConfigDir(); err == nil { // Unless the user has specifically overridden the configuration file
if info, err := os.Stat(configDir); err == nil && info.IsDir() { // location using an environment variable, we'll also load what we find
dirConfig, dirDiags := loadConfigDir(configDir) // in the config directory. We skip the config directory when source
diags = diags.Append(dirDiags) // file override is set because we interpret the environment variable
config = config.Merge(dirConfig) // being set as an intention to ignore the default set of CLI config
// files because we're doing something special, like running Terraform
// in automation with a locally-customized configuration.
if cliConfigFileOverride() == "" {
if configDir, err := ConfigDir(); err == nil {
if info, err := os.Stat(configDir); err == nil && info.IsDir() {
dirConfig, dirDiags := loadConfigDir(configDir)
diags = diags.Append(dirDiags)
config = config.Merge(dirConfig)
}
} }
} else {
log.Printf("[DEBUG] Not reading CLI config directory because config location is overridden by environment variable")
} }
if envConfig := EnvConfig(); envConfig != nil { if envConfig := EnvConfig(); envConfig != nil {
@ -136,6 +153,13 @@ func loadConfigFile(path string) (*Config, tfdiags.Diagnostics) {
return result, diags return result, diags
} }
// Deal with the provider_installation block, which is not handled using
// DecodeObject because its structure is not compatible with the
// limitations of that function.
providerInstBlocks, moreDiags := decodeProviderInstallationFromConfig(obj)
diags = diags.Append(moreDiags)
result.ProviderInstallation = providerInstBlocks
// Replace all env vars // Replace all env vars
for k, v := range result.Providers { for k, v := range result.Providers {
result.Providers[k] = os.ExpandEnv(v) result.Providers[k] = os.ExpandEnv(v)
@ -242,6 +266,13 @@ func (c *Config) Validate() tfdiags.Diagnostics {
) )
} }
// Should have zero or one "provider_installation" blocks
if len(c.ProviderInstallation) > 1 {
diags = diags.Append(
fmt.Errorf("No more than one provider_installation block may be specified"),
)
}
return diags return diags
} }
@ -310,17 +341,18 @@ func (c1 *Config) Merge(c2 *Config) *Config {
} }
} }
if (len(c1.ProviderInstallation) + len(c2.ProviderInstallation)) > 0 {
result.ProviderInstallation = append(result.ProviderInstallation, c1.ProviderInstallation...)
result.ProviderInstallation = append(result.ProviderInstallation, c2.ProviderInstallation...)
}
return &result return &result
} }
func cliConfigFile() (string, error) { func cliConfigFile() (string, error) {
mustExist := true mustExist := true
configFilePath := os.Getenv("TF_CLI_CONFIG_FILE") configFilePath := cliConfigFileOverride()
if configFilePath == "" {
configFilePath = os.Getenv("TERRAFORM_CONFIG")
}
if configFilePath == "" { if configFilePath == "" {
var err error var err error
configFilePath, err = ConfigFile() configFilePath, err = ConfigFile()
@ -347,3 +379,11 @@ func cliConfigFile() (string, error) {
log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.") log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.")
return "", nil return "", nil
} }
func cliConfigFileOverride() string {
configFilePath := os.Getenv("TF_CLI_CONFIG_FILE")
if configFilePath == "" {
configFilePath = os.Getenv("TERRAFORM_CONFIG")
}
return configFilePath
}

View File

@ -7,6 +7,7 @@ import (
"testing" "testing"
"github.com/davecgh/go-spew/spew" "github.com/davecgh/go-spew/spew"
"github.com/google/go-cmp/cmp"
) )
// This is the directory where our test fixtures are. // This is the directory where our test fixtures are.
@ -169,6 +170,29 @@ func TestConfigValidate(t *testing.T) {
}, },
1, // no more than one credentials_helper block allowed 1, // no more than one credentials_helper block allowed
}, },
"provider_installation good none": {
&Config{
ProviderInstallation: nil,
},
0,
},
"provider_installation good one": {
&Config{
ProviderInstallation: []*ProviderInstallation{
{},
},
},
0,
},
"provider_installation too many": {
&Config{
ProviderInstallation: []*ProviderInstallation{
{},
{},
},
},
1, // no more than one provider_installation block allowed
},
} }
for name, test := range tests { for name, test := range tests {
@ -209,6 +233,19 @@ func TestConfig_Merge(t *testing.T) {
CredentialsHelpers: map[string]*ConfigCredentialsHelper{ CredentialsHelpers: map[string]*ConfigCredentialsHelper{
"buz": {}, "buz": {},
}, },
ProviderInstallation: []*ProviderInstallation{
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("a")},
{Location: ProviderInstallationFilesystemMirror("b")},
},
},
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("c")},
},
},
},
} }
c2 := &Config{ c2 := &Config{
@ -234,6 +271,13 @@ func TestConfig_Merge(t *testing.T) {
CredentialsHelpers: map[string]*ConfigCredentialsHelper{ CredentialsHelpers: map[string]*ConfigCredentialsHelper{
"biz": {}, "biz": {},
}, },
ProviderInstallation: []*ProviderInstallation{
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("d")},
},
},
},
} }
expected := &Config{ expected := &Config{
@ -270,11 +314,29 @@ func TestConfig_Merge(t *testing.T) {
"buz": {}, "buz": {},
"biz": {}, "biz": {},
}, },
ProviderInstallation: []*ProviderInstallation{
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("a")},
{Location: ProviderInstallationFilesystemMirror("b")},
},
},
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("c")},
},
},
{
Methods: []*ProviderInstallationMethod{
{Location: ProviderInstallationFilesystemMirror("d")},
},
},
},
} }
actual := c1.Merge(c2) actual := c1.Merge(c2)
if !reflect.DeepEqual(actual, expected) { if diff := cmp.Diff(expected, actual); diff != "" {
t.Fatalf("bad: %#v", actual) t.Fatalf("wrong result\n%s", diff)
} }
} }

View File

@ -0,0 +1,265 @@
package cliconfig
import (
"fmt"
"github.com/hashicorp/hcl"
hclast "github.com/hashicorp/hcl/hcl/ast"
"github.com/hashicorp/terraform/tfdiags"
)
// ProviderInstallation is the structure of the "provider_installation"
// nested block within the CLI configuration.
type ProviderInstallation struct {
Methods []*ProviderInstallationMethod
}
// decodeProviderInstallationFromConfig uses the HCL AST API directly to
// decode "provider_installation" blocks from the given file.
//
// This uses the HCL AST directly, rather than HCL's decoder, because the
// intended configuration structure can't be represented using the HCL
// decoder's struct tags. This structure is intended as something that would
// be relatively easier to deal with in HCL 2 once we eventually migrate
// CLI config over to that, and so this function is stricter than HCL 1's
// decoder would be in terms of exactly what configuration shape it is
// expecting.
//
// Note that this function wants the top-level file object which might or
// might not contain provider_installation blocks, not a provider_installation
// block directly itself.
func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInstallation, tfdiags.Diagnostics) {
var ret []*ProviderInstallation
var diags tfdiags.Diagnostics
root := hclFile.Node.(*hclast.ObjectList)
// This is a rather odd hybrid: it's a HCL 2-like decode implemented using
// the HCL 1 AST API. That makes it a bit awkward in places, but it allows
// us to mimick the strictness of HCL 2 (making a later migration easier)
// and to support a block structure that the HCL 1 decoder can't represent.
for _, block := range root.Items {
if block.Keys[0].Token.Value() != "provider_installation" {
continue
}
// HCL only tracks whether the input was JSON or native syntax inside
// individual tokens, so we'll use our block type token to decide
// and assume that the rest of the block must be written in the same
// syntax, because syntax is a whole-file idea.
isJSON := block.Keys[0].Token.JSON
if block.Assign.Line != 0 && !isJSON {
// Seems to be an attribute rather than a block
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation block",
fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()),
))
continue
}
if len(block.Keys) > 1 && !isJSON {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation block",
fmt.Sprintf("The provider_installation block at %s must not have any labels.", block.Pos()),
))
}
pi := &ProviderInstallation{}
body, ok := block.Val.(*hclast.ObjectType)
if !ok {
// We can't get in here with native HCL syntax because we
// already checked above that we're using block syntax, but
// if we're reading JSON then our value could potentially be
// anything.
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation block",
fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()),
))
continue
}
for _, methodBlock := range body.List.Items {
if methodBlock.Assign.Line != 0 && !isJSON {
// Seems to be an attribute rather than a block
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()),
))
continue
}
if len(methodBlock.Keys) > 1 && !isJSON {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("The blocks inside the provider_installation block at %s may not have any labels.", block.Pos()),
))
}
methodBody, ok := methodBlock.Val.(*hclast.ObjectType)
if !ok {
// We can't get in here with native HCL syntax because we
// already checked above that we're using block syntax, but
// if we're reading JSON then our value could potentially be
// anything.
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()),
))
continue
}
methodTypeStr := methodBlock.Keys[0].Token.Value().(string)
var location ProviderInstallationLocation
var include, exclude []string
switch methodTypeStr {
case "direct":
type BodyContent struct {
Include []string `hcl:"include"`
Exclude []string `hcl:"exclude"`
}
var bodyContent BodyContent
err := hcl.DecodeObject(&bodyContent, methodBody)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err),
))
continue
}
location = ProviderInstallationDirect
include = bodyContent.Include
exclude = bodyContent.Exclude
case "filesystem_mirror":
type BodyContent struct {
Path string `hcl:"path"`
Include []string `hcl:"include"`
Exclude []string `hcl:"exclude"`
}
var bodyContent BodyContent
err := hcl.DecodeObject(&bodyContent, methodBody)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err),
))
continue
}
if bodyContent.Path == "" {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Invalid %s block at %s: \"path\" argument is required.", methodTypeStr, block.Pos()),
))
continue
}
location = ProviderInstallationFilesystemMirror(bodyContent.Path)
include = bodyContent.Include
exclude = bodyContent.Exclude
case "network_mirror":
type BodyContent struct {
URL string `hcl:"url"`
Include []string `hcl:"include"`
Exclude []string `hcl:"exclude"`
}
var bodyContent BodyContent
err := hcl.DecodeObject(&bodyContent, methodBody)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err),
))
continue
}
if bodyContent.URL == "" {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Invalid %s block at %s: \"url\" argument is required.", methodTypeStr, block.Pos()),
))
continue
}
location = ProviderInstallationNetworkMirror(bodyContent.URL)
include = bodyContent.Include
exclude = bodyContent.Exclude
default:
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider_installation method block",
fmt.Sprintf("Unknown provider installation method %q at %s.", methodTypeStr, methodBlock.Pos()),
))
continue
}
pi.Methods = append(pi.Methods, &ProviderInstallationMethod{
Location: location,
Include: include,
Exclude: exclude,
})
}
ret = append(ret, pi)
}
return ret, diags
}
// ProviderInstallationMethod represents an installation method block inside
// a provider_installation block.
type ProviderInstallationMethod struct {
Location ProviderInstallationLocation
Include []string `hcl:"include"`
Exclude []string `hcl:"exclude"`
}
// ProviderInstallationLocation is an interface type representing the
// different installation location types. The concrete implementations of
// this interface are:
//
// ProviderInstallationDirect: install from the provider's origin registry
// ProviderInstallationFilesystemMirror(dir): install from a local filesystem mirror
// ProviderInstallationNetworkMirror(host): install from a network mirror
type ProviderInstallationLocation interface {
providerInstallationLocation()
}
type providerInstallationDirect [0]byte
func (i providerInstallationDirect) providerInstallationLocation() {}
// ProviderInstallationDirect is a ProviderInstallationSourceLocation
// representing installation from a provider's origin registry.
var ProviderInstallationDirect ProviderInstallationLocation = providerInstallationDirect{}
func (i providerInstallationDirect) GoString() string {
return "cliconfig.ProviderInstallationDirect"
}
// ProviderInstallationFilesystemMirror is a ProviderInstallationSourceLocation
// representing installation from a particular local filesystem mirror. The
// string value is the filesystem path to the mirror directory.
type ProviderInstallationFilesystemMirror string
func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {}
func (i ProviderInstallationFilesystemMirror) GoString() string {
return fmt.Sprintf("cliconfig.ProviderInstallationFilesystemMirror(%q)", i)
}
// ProviderInstallationNetworkMirror is a ProviderInstallationSourceLocation
// representing installation from a particular local network mirror. The
// string value is the HTTP base URL exactly as written in the configuration,
// without any normalization.
type ProviderInstallationNetworkMirror string
func (i ProviderInstallationNetworkMirror) providerInstallationLocation() {}
func (i ProviderInstallationNetworkMirror) GoString() string {
return fmt.Sprintf("cliconfig.ProviderInstallationNetworkMirror(%q)", i)
}

View File

@ -0,0 +1,70 @@
package cliconfig
import (
"path/filepath"
"testing"
"github.com/google/go-cmp/cmp"
)
func TestLoadConfig_providerInstallation(t *testing.T) {
for _, configFile := range []string{"provider-installation", "provider-installation.json"} {
t.Run(configFile, func(t *testing.T) {
got, diags := loadConfigFile(filepath.Join(fixtureDir, configFile))
if diags.HasErrors() {
t.Errorf("unexpected diagnostics: %s", diags.Err().Error())
}
want := &Config{
ProviderInstallation: []*ProviderInstallation{
{
Methods: []*ProviderInstallationMethod{
{
Location: ProviderInstallationFilesystemMirror("/tmp/example1"),
Include: []string{"example.com/*/*"},
},
{
Location: ProviderInstallationNetworkMirror("https://tf-Mirror.example.com/"),
Include: []string{"registry.terraform.io/*/*"},
Exclude: []string{"registry.Terraform.io/foobar/*"},
},
{
Location: ProviderInstallationFilesystemMirror("/tmp/example2"),
},
{
Location: ProviderInstallationDirect,
Exclude: []string{"example.com/*/*"},
},
},
},
},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("wrong result\n%s", diff)
}
})
}
}
func TestLoadConfig_providerInstallationErrors(t *testing.T) {
_, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation-errors"))
want := `7 problems:
- Invalid provider_installation method block: Unknown provider installation method "not_a_thing" at 2:3.
- Invalid provider_installation method block: Invalid filesystem_mirror block at 1:1: "path" argument is required.
- Invalid provider_installation method block: Invalid network_mirror block at 1:1: "url" argument is required.
- Invalid provider_installation method block: The items inside the provider_installation block at 1:1 must all be blocks.
- Invalid provider_installation method block: The blocks inside the provider_installation block at 1:1 may not have any labels.
- Invalid provider_installation block: The provider_installation block at 9:1 must not have any labels.
- Invalid provider_installation block: The provider_installation block at 11:1 must not be introduced with an equals sign.`
// The above error messages include only line/column location information
// and not file location information because HCL 1 does not store
// information about the filename a location belongs to. (There is a field
// for it in token.Pos but it's always an empty string in practice.)
if got := diags.Err().Error(); got != want {
t.Errorf("wrong diagnostics\ngot:\n%s\nwant:\n%s", got, want)
}
}

View File

@ -0,0 +1,17 @@
provider_installation {
filesystem_mirror {
path = "/tmp/example1"
include = ["example.com/*/*"]
}
network_mirror {
url = "https://tf-Mirror.example.com/"
include = ["registry.terraform.io/*/*"]
exclude = ["registry.Terraform.io/foobar/*"]
}
filesystem_mirror {
path = "/tmp/example2"
}
direct {
exclude = ["example.com/*/*"]
}
}

View File

@ -0,0 +1,11 @@
provider_installation {
not_a_thing {} # unknown source type
filesystem_mirror {} # missing "path" argument
network_mirror {} # missing "host" argument
direct = {} # should be a block, not an argument
direct "what" {} # should not have a label
}
provider_installation "what" {} # should not have a label
provider_installation = {} # should be a block, not an argument

View File

@ -0,0 +1,19 @@
{
"provider_installation": {
"filesystem_mirror": [{
"path": "/tmp/example1",
"include": ["example.com/*/*"]
}],
"network_mirror": [{
"url": "https://tf-Mirror.example.com/",
"include": ["registry.terraform.io/*/*"],
"exclude": ["registry.Terraform.io/foobar/*"]
}],
"filesystem_mirror": [{
"path": "/tmp/example2"
}],
"direct": [{
"exclude": ["example.com/*/*"]
}]
}
}

View File

@ -173,6 +173,56 @@ func TestInitProvidersLocalOnly(t *testing.T) {
} }
} }
func TestInitProvidersCustomMethod(t *testing.T) {
t.Parallel()
// This test should not reach out to the network if it is behaving as
// intended. If it _does_ try to access an upstream registry and encounter
// an error doing so then that's a legitimate test failure that should be
// fixed. (If it incorrectly reaches out anywhere then it's likely to be
// to the host "example.com", which is the placeholder domain we use in
// the test fixture.)
for _, configFile := range []string{"cliconfig.tfrc", "cliconfig.tfrc.json"} {
t.Run(configFile, func(t *testing.T) {
fixturePath := filepath.Join("testdata", "custom-provider-install-method")
tf := e2e.NewBinary(terraformBin, fixturePath)
defer tf.Close()
// Our fixture dir has a generic os_arch dir, which we need to customize
// to the actual OS/arch where this test is running in order to get the
// desired result.
fixtMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch")
wantMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
err := os.Rename(fixtMachineDir, wantMachineDir)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
// We'll use a local CLI configuration file taken from our fixture
// directory so we can force a custom installation method config.
tf.AddEnv("TF_CLI_CONFIG_FILE=" + tf.Path(configFile))
stdout, stderr, err := tf.Run("init")
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if stderr != "" {
t.Errorf("unexpected stderr output:\n%s", stderr)
}
if !strings.Contains(stdout, "Terraform has been successfully initialized!") {
t.Errorf("success message is missing from output:\n%s", stdout)
}
if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") {
t.Errorf("provider download message is missing from output:\n%s", stdout)
}
})
}
}
func TestInitProviders_pluginCache(t *testing.T) { func TestInitProviders_pluginCache(t *testing.T) {
t.Parallel() t.Parallel()

View File

@ -0,0 +1,8 @@
provider_installation {
filesystem_mirror {
path = "./fs-mirror"
}
direct {
exclude = ["example.com/*/*"]
}
}

View File

@ -0,0 +1,9 @@
{
"provider_installation": {
"filesystem_mirror": [
{
"path": "./fs-mirror"
}
]
}
}

View File

@ -0,0 +1,2 @@
This is not a real plugin executable. It's just here to be discovered by the
provider installation process.

View File

@ -0,0 +1,21 @@
# The purpose of this test is to refer to a provider whose address contains
# a hostname that is only used for namespacing purposes and doesn't actually
# have a provider registry deployed at it.
#
# A user can install such a provider in one of the implied local filesystem
# directories and Terraform should accept that as the selection for that
# provider without producing any errors about the fact that example.com
# does not have a provider registry.
#
# For this test in particular we're using the "vendor" directory that is
# the documented way to include provider plugins directly inside a
# configuration uploaded to Terraform Cloud, but this functionality applies
# to all of the implicit local filesystem search directories.
terraform {
required_providers {
happycloud = {
source = "example.com/awesomecorp/happycloud"
}
}
}

View File

@ -1,6 +1,7 @@
package getproviders package getproviders
import ( import (
"fmt"
"net/url" "net/url"
"github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/addrs"
@ -26,13 +27,11 @@ func NewHTTPMirrorSource(baseURL *url.URL) *HTTPMirrorSource {
// AvailableVersions retrieves the available versions for the given provider // AvailableVersions retrieves the available versions for the given provider
// from the object's underlying HTTP mirror service. // from the object's underlying HTTP mirror service.
func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) { func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) {
// TODO: Implement return nil, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform")
panic("HTTPMirrorSource.AvailableVersions not yet implemented")
} }
// PackageMeta retrieves metadata for the requested provider package // PackageMeta retrieves metadata for the requested provider package
// from the object's underlying HTTP mirror service. // from the object's underlying HTTP mirror service.
func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) {
// TODO: Implement return PackageMeta{}, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform")
panic("HTTPMirrorSource.PackageMeta not yet implemented")
} }

22
main.go
View File

@ -127,6 +127,7 @@ func wrappedMain() int {
log.Printf("[INFO] CLI args: %#v", os.Args) log.Printf("[INFO] CLI args: %#v", os.Args)
config, diags := cliconfig.LoadConfig() config, diags := cliconfig.LoadConfig()
if len(diags) > 0 { if len(diags) > 0 {
// Since we haven't instantiated a command.Meta yet, we need to do // Since we haven't instantiated a command.Meta yet, we need to do
// some things manually here and use some "safe" defaults for things // some things manually here and use some "safe" defaults for things
@ -164,11 +165,22 @@ func wrappedMain() int {
services := disco.NewWithCredentialsSource(credsSrc) services := disco.NewWithCredentialsSource(credsSrc)
services.SetUserAgent(httpclient.TerraformUserAgent(version.String())) services.SetUserAgent(httpclient.TerraformUserAgent(version.String()))
// For the moment, we just always use the registry source to install providerSrc, diags := providerSource(config.ProviderInstallation, services)
// direct from a registry. In future there should be a mechanism to if len(diags) > 0 {
// configure providers sources from the CLI config, which will then Ui.Error("There are some problems with the provider_installation configuration:")
// change how we construct this object. for _, diag := range diags {
providerSrc := providerSource(services) earlyColor := &colorstring.Colorize{
Colors: colorstring.DefaultColors,
Disable: true, // Disable color to be conservative until we know better
Reset: true,
}
Ui.Error(format.Diagnostic(diag, nil, earlyColor, 78))
}
if diags.HasErrors() {
Ui.Error("As a result of the above problems, Terraform's provider installer may not behave as intended.\n\n")
// We continue to run anyway, because most commands don't do provider installation.
}
}
// Initialize the backends. // Initialize the backends.
backendInit.Init(services) backendInit.Init(services)

View File

@ -1,7 +1,9 @@
package main package main
import ( import (
"fmt"
"log" "log"
"net/url"
"os" "os"
"path/filepath" "path/filepath"
@ -11,18 +13,69 @@ import (
"github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/addrs"
"github.com/hashicorp/terraform/command/cliconfig" "github.com/hashicorp/terraform/command/cliconfig"
"github.com/hashicorp/terraform/internal/getproviders" "github.com/hashicorp/terraform/internal/getproviders"
"github.com/hashicorp/terraform/tfdiags"
) )
// providerSource constructs a provider source based on a combination of the // providerSource constructs a provider source based on a combination of the
// CLI configuration and some default search locations. This will be the // CLI configuration and some default search locations. This will be the
// provider source used for provider installation in the "terraform init" // provider source used for provider installation in the "terraform init"
// command, unless overridden by the special -plugin-dir option. // command, unless overridden by the special -plugin-dir option.
func providerSource(services *disco.Disco) getproviders.Source { func providerSource(configs []*cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) {
// We're not yet using the CLI config here because we've not implemented if len(configs) == 0 {
// yet the new configuration constructs to customize provider search // If there's no explicit installation configuration then we'll build
// locations. That'll come later. For now, we just always use the // up an implicit one with direct registry installation along with
// implicit default provider source. // some automatically-selected local filesystem mirrors.
return implicitProviderSource(services) return implicitProviderSource(services), nil
}
// There should only be zero or one configurations, which is checked by
// the validation logic in the cliconfig package. Therefore we'll just
// ignore any additional configurations in here.
config := configs[0]
return explicitProviderSource(config, services)
}
func explicitProviderSource(config *cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
var searchRules []getproviders.MultiSourceSelector
log.Printf("[DEBUG] Explicit provider installation configuration is set")
for _, methodConfig := range config.Methods {
source, moreDiags := providerSourceForCLIConfigLocation(methodConfig.Location, services)
diags = diags.Append(moreDiags)
if moreDiags.HasErrors() {
continue
}
include, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider source inclusion patterns",
fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err),
))
continue
}
exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Exclude)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid provider source exclusion patterns",
fmt.Sprintf("CLI config specifies invalid provider exclusion patterns: %s.", err),
))
continue
}
searchRules = append(searchRules, getproviders.MultiSourceSelector{
Source: source,
Include: include,
Exclude: exclude,
})
log.Printf("[TRACE] Selected provider installation method %#v with includes %s and excludes %s", methodConfig.Location, include, exclude)
}
return getproviders.MultiSource(searchRules), diags
} }
// implicitProviderSource builds a default provider source to use if there's // implicitProviderSource builds a default provider source to use if there's
@ -130,3 +183,45 @@ func implicitProviderSource(services *disco.Disco) getproviders.Source {
return getproviders.MultiSource(searchRules) return getproviders.MultiSource(searchRules)
} }
func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationLocation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) {
if loc == cliconfig.ProviderInstallationDirect {
return getproviders.NewMemoizeSource(
getproviders.NewRegistrySource(services),
), nil
}
switch loc := loc.(type) {
case cliconfig.ProviderInstallationFilesystemMirror:
return getproviders.NewFilesystemMirrorSource(string(loc)), nil
case cliconfig.ProviderInstallationNetworkMirror:
url, err := url.Parse(string(loc))
if err != nil {
var diags tfdiags.Diagnostics
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid URL for provider installation source",
fmt.Sprintf("Cannot parse %q as a URL for a network provider mirror: %s.", string(loc), err),
))
return nil, diags
}
if url.Scheme != "https" || url.Host == "" {
var diags tfdiags.Diagnostics
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid URL for provider installation source",
fmt.Sprintf("Cannot use %q as a URL for a network provider mirror: the mirror must be at an https: URL.", string(loc)),
))
return nil, diags
}
return getproviders.NewHTTPMirrorSource(url), nil
default:
// We should not get here because the set of cases above should
// be comprehensive for all of the
// cliconfig.ProviderInstallationLocation implementations.
panic(fmt.Sprintf("unexpected provider source location type %T", loc))
}
}

View File

@ -51,6 +51,14 @@ disable_checkpoint = true
The following settings can be set in the CLI configuration file: The following settings can be set in the CLI configuration file:
- `credentials` - configures credentials for use with Terraform Cloud or
Terraform Enterprise. See [Credentials](#credentials) below for more
information.
- `credentials_helper` - configures an external helper program for the storage
and retrieval of credentials for Terraform Cloud or Terraform Enterprise.
See [Credentials Helpers](#credentials-helpers) below for more information.
- `disable_checkpoint` — when set to `true`, disables - `disable_checkpoint` — when set to `true`, disables
[upgrade and security bulletin checks](/docs/commands/index.html#upgrade-and-security-bulletin-checks) [upgrade and security bulletin checks](/docs/commands/index.html#upgrade-and-security-bulletin-checks)
that require reaching out to HashiCorp-provided network services. that require reaching out to HashiCorp-provided network services.
@ -60,16 +68,12 @@ The following settings can be set in the CLI configuration file:
id used to de-duplicate warning messages. id used to de-duplicate warning messages.
- `plugin_cache_dir` — enables - `plugin_cache_dir` — enables
[plugin caching](/docs/configuration/providers.html#provider-plugin-cache) [plugin caching](#provider-plugin-cache)
and specifies, as a string, the location of the plugin cache directory. and specifies, as a string, the location of the plugin cache directory.
- `credentials` - configures credentials for use with Terraform Cloud or - `provider_installation` - customizes the installation methods used by
Terraform Enterprise. See [Credentials](#credentials) below for more `terraform init` when installing provider plugins. See
information. [Provider Installation](#provider-installation) below for more information.
- `credentials_helper` - configures an external helper program for the storage
and retrieval of credentials for Terraform Cloud or Terraform Enterprise.
See [Credentials Helpers](#credentials-helpers) below for more information.
## Credentials ## Credentials
@ -144,14 +148,200 @@ To learn how to write and install your own credentials helpers to integrate
with existing in-house credentials management systems, see with existing in-house credentials management systems, see
[the guide to Credentials Helper internals](/docs/internals/credentials-helpers.html). [the guide to Credentials Helper internals](/docs/internals/credentials-helpers.html).
## Deprecated Settings ## Provider Installation
The following settings are supported for backward compatibility but are no The default way to install provider plugins is from a provider registry. The
longer recommended for use: origin registry for a provider is encoded in the provider's source address,
like `registry.terraform.io/hashicorp/aws`. For convenience in the common case,
Terraform allows omitting the hostname portion for providers on
`registry.terraform.io`, so we'd normally write `hashicorp/aws` instead in
this case.
Downloading a plugin directly from its origin registry is not always
appropriate, though. For example, the system where you are running Terraform
may not be able to access an origin registry due to firewall restrictions
within your organization or your locality.
To allow using Terraform providers in these situations, there are some
alternative options for making provider plugins available to Terraform which
we'll describe in the following sections.
### Explicit Installation Method Configuration
A `provider_installation` block in the CLI configuration allows overriding
Terraform's default installation behaviors, so you can force Terraform to use
a local mirror for some or all of the providers you intend to use.
The general structure of a `provider_installation` block is as follows:
```hcl
provider_installation {
filesystem_mirror {
path = "/usr/share/terraform/providers"
include = ["example.com/*/*"]
}
direct {
exclude = ["example.com/*/*"]
}
}
```
Each of the nested blocks inside the `provider_installation` block specifies
one installation method. Each installation method can take both `include`
and `exclude` patterns that specify which providers a particular installation
method can be used for. In the example above, we specify that any provider
whose origin registry is at `example.com` can be installed only from the
filesystem mirror at `/usr/share/terraform/providers`, while all other
providers can be installed only directly from their origin registries.
If you set both both `include` and `exclude` for a particular installation
method, the exclusion patterns take priority. For example, including
`registry.terraform.io/hashicorp/*` but also excluding
`registry.terraform.io/hashicorp/dns` will make that installation method apply
to everything in the `hashicorp` namespace with the exception of
`hashicorp/dns`.
As with provider source addresses in the main configuration, you can omit
the `registry.terraform.io/` prefix for providers distributed through the
public Terraform registry, even when using wildcards. For example,
`registry.terraform.io/hashicorp/*` and `hashicorp/*` are equivalent.
`*/*` is a shorthand for `registry.terraform.io/*/*`, not for
`*/*/*`.
The following are the two supported installation method types:
* `direct`: request information about the provider directly from its origin
registry and download over the network from the location that registry
indicates. This method expects no additional arguments.
* `filesystem_mirror`: consult a directory on the local disk for copies of
providers. This method requires the additional argument `path` to indicate
which directory to look in.
Terraform expects the given directory to contain a nested directory structure
where the path segments together provide metadata about the available
providers. The following two directory structures are supported:
* Packed layout: `HOSTNAME/NAMESPACE/TYPE/terraform-provider-TYPE_VERSION_TARGET.zip`
is the distribution zip file obtained from the provider's origin registry.
* Unpacked layout: `HOSTNAME/NAMESPACE/TYPE/VERSION/TARGET` is a directory
containing the result of extracting the provider's distribution zip file.
In both layouts, the `VERSION` is a string like `2.0.0` and the `TARGET`
specifies a particular target platform using a format like `darwin_amd64`,
`linux_arm`, `windows_amd64`, etc.
If you use the unpacked layout, Terraform will attempt to create a symbolic
link to the mirror directory when installing the provider, rather than
creating a deep copy of the directory. The packed layout prevents this
because Terraform must extract the zip file during installation.
You can include multiple `filesystem_mirror` blocks in order to specify
several different directories to search.
Terraform will try all of the specified methods whose include and exclude
patterns match a given provider, and select the newest version available across
all of those methods that matches the version constraint given in each
Terraform configuration. If you have a local mirror of a particular provider
and intend Terraform to use that local mirror exclusively, you must either
remove the `direct` installation method altogether or use its `exclude`
argument to disable its use for specific providers.
### Implied Local Mirror Directories
If your CLI configuration does not include a `provider_installation` block at
all, Terraform produces an _implied_ configuration. The implied configuration
includes a selection of `filesystem_mirror` methods and then the `direct`
method.
The set of directories Terraform can select as filesystem mirrors depends on
the operating system where you are running Terraform:
* **Windows:** `%APPDATA%/HashiCorp/Terraform/plugins`
* **Mac OS X:** `~/Library/Application Support/io.terraform/plugins` and
`/Library/Application Support/io.terraform/plugins`
* **Linux and other Unix-like systems**: Terraform implements the
[XDG Base Directory](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html)
specification and appends `terraform/plugins` to all of the specified
data directories. Without any XDG environment variables set, Terraform
will use `~/.local/share/terraform/plugins`,
`/usr/local/share/terraform/plugins`, and `/usr/share/terraform/plugins`.
Terraform will create an implied `filesystem_mirror` method block for each of
the directories indicated above that exists when Terraform starts up.
In addition to the zero or more implied `filesystem_mirror` blocks, Terraform
also creates an implied `direct` block. Terraform will scan all of the
filesystem mirror directories to see which providers are placed there and
automatically exclude all of those providers from the implied `direct` block.
(This automatic `exclude` behavior applies only to _implicit_ `direct` blocks;
if you use explicit `provider_installation` you will need to write the intended
exclusions out yourself.)
### Provider Plugin Cache
By default, `terraform init` downloads plugins into a subdirectory of the
working directory so that each working directory is self-contained. As a
consequence, if you have multiple configurations that use the same provider
then a separate copy of its plugin will be downloaded for each configuration.
Given that provider plugins can be quite large (on the order of hundreds of
megabytes), this default behavior can be inconvenient for those with slow
or metered Internet connections. Therefore Terraform optionally allows the
use of a local directory as a shared plugin cache, which then allows each
distinct plugin binary to be downloaded only once.
To enable the plugin cache, use the `plugin_cache_dir` setting in
the CLI configuration file. For example:
```hcl
plugin_cache_dir = "$HOME/.terraform.d/plugin-cache"
```
This directory must already exist before Terraform will cache plugins;
Terraform will not create the directory itself.
Please note that on Windows it is necessary to use forward slash separators
(`/`) rather than the conventional backslash (`\`) since the configuration
file parser considers a backslash to begin an escape sequence.
Setting this in the configuration file is the recommended approach for a
persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment
variable can be used to enable caching or to override an existing cache
directory within a particular shell session:
```bash
export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache"
```
When a plugin cache directory is enabled, the `terraform init` command will
still use the configured or implied installation methods to obtain metadata
about which plugins are available, but once a suitable version has been
selected it will first check to see if the chosen plugin is already available
in the cache directory. If so, Terraform will use the previously-downloaded
copy.
If the selected plugin is not already in the cache, Terraform will download
it into the cache first and then copy it from there into the correct location
under your current working directory. When possible Terraform will use
symbolic links to avoid storing a separate copy of a cached plugin in multiple
directories.
The plugin cache directory _must not_ also be one of the configured or implied
filesystem mirror directories, since the cache management logic conflicts with
the filesystem mirror logic when operating on the same directory.
Terraform will never itself delete a plugin from the plugin cache once it has
been placed there. Over time, as plugins are upgraded, the cache directory may
grow to contain several unused versions which you must delete manually.
## Removed Settings
The following settings are supported in Terraform 0.12 and earlier but are
no longer recommended for use:
* `providers` - a configuration block that allows specifying the locations of * `providers` - a configuration block that allows specifying the locations of
specific plugins for each named provider. This mechanism is deprecated specific plugins for each named provider. This mechanism is deprecated
because it is unable to specify a version number for each plugin, and thus because it is unable to specify a version number and source for each provider.
it does not co-operate with the plugin versioning mechanism. Instead, See [Provider Installation](#provider-installation) above for the replacement
place the plugin executable files in of this setting in Terraform 0.13 and later.
[the third-party plugins directory](/docs/configuration/providers.html#third-party-plugins).

View File

@ -233,65 +233,35 @@ from their parents.
Anyone can develop and distribute their own Terraform providers. (See Anyone can develop and distribute their own Terraform providers. (See
[Writing Custom Providers](/docs/extend/writing-custom-providers.html) for more [Writing Custom Providers](/docs/extend/writing-custom-providers.html) for more
about provider development.) These third-party providers must be manually about provider development.)
installed, since `terraform init` cannot automatically download them.
Install third-party providers by placing their plugin executables in the user The main way to distribute a provider is via a provider registry, and the main
plugins directory. The user plugins directory is in one of the following provider registry is
locations, depending on the host operating system: [part of the public Terraform Registry](https://registry.terraform.io/browse/providers),
along with public shared modules.
Operating system | User plugins directory Providers distributed via a public registry to not require any special
------------------|----------------------- additional configuration to use, once you know their source addresses. You can
Windows | `%APPDATA%\terraform.d\plugins` specify both official and third-party source addresses in the
All other systems | `~/.terraform.d/plugins` `required_providers` block in your module:
Once a plugin is installed, `terraform init` can initialize it normally. You must run this command from the directory where the configuration files are located. ```hcl
terraform {
required_providers {
# An example third-party provider. Not actually available.
example = {
source = "example.com/examplecorp/example"
}
}
}
```
Providers distributed by HashiCorp can also go in the user plugins directory. If Installing directly from a registry is not appropriate for all situations,
a manually installed version meets the configuration's version constraints, though. If you are running Terraform from a system that cannot access some or
Terraform will use it instead of downloading that provider. This is useful in all of the necessary origin registries, you can configure Terraform to obtain
airgapped environments and when testing pre-release provider builds. providers from a local mirror instead. For more information, see
[Provider Installation](../commands/cli-config.html#provider-installation)
### Plugin Names and Versions in the CLI configuration documentation.
The naming scheme for provider plugins is `terraform-provider-<NAME>_vX.Y.Z`,
and Terraform uses the name to understand the name and version of a particular
provider binary.
If multiple versions of a plugin are installed, Terraform will use the newest
version that meets the configuration's version constraints.
Third-party plugins are often distributed with an appropriate filename already
set in the distribution archive, so that they can be extracted directly into the
user plugins directory.
### OS and Architecture Directories
Terraform plugins are compiled for a specific operating system and architecture,
and any plugins in the root of the user plugins directory must be compiled for
the current system.
If you use the same plugins directory on multiple systems, you can install
plugins into subdirectories with a naming scheme of `<OS>_<ARCH>` (for example,
`darwin_amd64`). Terraform uses plugins from the root of the plugins directory
and from the subdirectory that corresponds to the current system, ignoring
other subdirectories.
Terraform's OS and architecture strings are the standard ones used by the Go
language. The following are the most common:
* `darwin_amd64`
* `freebsd_386`
* `freebsd_amd64`
* `freebsd_arm`
* `linux_386`
* `linux_amd64`
* `linux_arm`
* `openbsd_386`
* `openbsd_amd64`
* `solaris_amd64`
* `windows_386`
* `windows_amd64`
## Provider Plugin Cache ## Provider Plugin Cache
@ -308,51 +278,3 @@ distinct plugin binary to be downloaded only once.
To enable the plugin cache, use the `plugin_cache_dir` setting in To enable the plugin cache, use the `plugin_cache_dir` setting in
[the CLI configuration file](/docs/commands/cli-config.html). [the CLI configuration file](/docs/commands/cli-config.html).
For example:
```hcl
# (Note that the CLI configuration file is _not_ the same as the .tf files
# used to configure infrastructure.)
plugin_cache_dir = "$HOME/.terraform.d/plugin-cache"
```
This directory must already exist before Terraform will cache plugins;
Terraform will not create the directory itself.
Please note that on Windows it is necessary to use forward slash separators
(`/`) rather than the conventional backslash (`\`) since the configuration
file parser considers a backslash to begin an escape sequence.
Setting this in the configuration file is the recommended approach for a
persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment
variable can be used to enable caching or to override an existing cache
directory within a particular shell session:
```bash
export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache"
```
When a plugin cache directory is enabled, the `terraform init` command will
still access the plugin distribution server to obtain metadata about which
plugins are available, but once a suitable version has been selected it will
first check to see if the selected plugin is already available in the cache
directory. If so, the already-downloaded plugin binary will be used.
If the selected plugin is not already in the cache, it will be downloaded
into the cache first and then copied from there into the correct location
under your current working directory.
When possible, Terraform will use hardlinks or symlinks to avoid storing
a separate copy of a cached plugin in multiple directories. At present, this
is not supported on Windows and instead a copy is always created.
The plugin cache directory must _not_ be the third-party plugin directory
or any other directory Terraform searches for pre-installed plugins, since
the cache management logic conflicts with the normal plugin discovery logic
when operating on the same directory.
Please note that Terraform will never itself delete a plugin from the
plugin cache once it's been placed there. Over time, as plugins are upgraded,
the cache directory may grow to contain several unused versions which must be
manually deleted.