From 2c535d829d7b92007b83493b56b9ffbbe1f8b448 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Apr 2020 15:48:07 -0700 Subject: [PATCH 01/14] command/cliconfig: Decode provider_installation blocks This new CLI config block type allows explicitly specifying where Terraform should look to find provider plugins for installation. This is not used anywhere as of this commit, but in a future commit we'll change package main to treat the presence of a block of this type as a request to disable the default set of provider sources and use these explicitly- specified ones instead. --- command/cliconfig/cliconfig.go | 43 +++- command/cliconfig/cliconfig_test.go | 66 ++++- command/cliconfig/provider_installation.go | 236 ++++++++++++++++++ .../cliconfig/provider_installation_test.go | 66 +++++ .../cliconfig/testdata/provider-installation | 17 ++ .../testdata/provider-installation-errors | 11 + 6 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 command/cliconfig/provider_installation.go create mode 100644 command/cliconfig/provider_installation_test.go create mode 100644 command/cliconfig/testdata/provider-installation create mode 100644 command/cliconfig/testdata/provider-installation-errors diff --git a/command/cliconfig/cliconfig.go b/command/cliconfig/cliconfig.go index ea0bf1e57..e36b11cc5 100644 --- a/command/cliconfig/cliconfig.go +++ b/command/cliconfig/cliconfig.go @@ -17,7 +17,7 @@ import ( "github.com/hashicorp/hcl" - "github.com/hashicorp/terraform-svchost" + svchost "github.com/hashicorp/terraform-svchost" "github.com/hashicorp/terraform/tfdiags" ) @@ -42,6 +42,12 @@ type Config struct { Credentials map[string]map[string]interface{} `hcl:"credentials"` CredentialsHelpers map[string]*ConfigCredentialsHelper `hcl:"credentials_helper"` + + // ProviderInstallation represents any provider_installation blocks + // in the configuration. Only one of these is allowed across the whole + // configuration, but we decode into a slice here so that we can handle + // that validation at validation time rather than initial decode time. + ProviderInstallation []*ProviderInstallation } // ConfigHost is the structure of the "host" nested block within the CLI @@ -57,6 +63,22 @@ type ConfigCredentialsHelper struct { Args []string `hcl:"args"` } +// ConfigProviderInstallationFilesystemMirror represents a "filesystem_mirror" +// block inside ConfigProviderInstallation. +type ConfigProviderInstallationFilesystemMirror struct { + Path string `hcl:"path"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` +} + +// ConfigProviderInstallationNetworkMirror represents a "network_mirror" block +// inside ConfigProviderInstallation. +type ConfigProviderInstallationNetworkMirror struct { + Hostname string `hcl:"hostname"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` +} + // BuiltinConfig is the built-in defaults for the configuration. These // can be overridden by user configurations. var BuiltinConfig Config @@ -136,6 +158,13 @@ func loadConfigFile(path string) (*Config, tfdiags.Diagnostics) { return result, diags } + // Deal with the provider_installation block, which is not handled using + // DecodeObject because its structure is not compatible with the + // limitations of that function. + providerInstBlocks, moreDiags := decodeProviderInstallationFromConfig(obj) + diags = diags.Append(moreDiags) + result.ProviderInstallation = providerInstBlocks + // Replace all env vars for k, v := range result.Providers { result.Providers[k] = os.ExpandEnv(v) @@ -242,6 +271,13 @@ func (c *Config) Validate() tfdiags.Diagnostics { ) } + // Should have zero or one "provider_installation" blocks + if len(c.ProviderInstallation) > 1 { + diags = diags.Append( + fmt.Errorf("No more than one provider_installation block may be specified"), + ) + } + return diags } @@ -310,6 +346,11 @@ func (c1 *Config) Merge(c2 *Config) *Config { } } + if (len(c1.ProviderInstallation) + len(c2.ProviderInstallation)) > 0 { + result.ProviderInstallation = append(result.ProviderInstallation, c1.ProviderInstallation...) + result.ProviderInstallation = append(result.ProviderInstallation, c2.ProviderInstallation...) + } + return &result } diff --git a/command/cliconfig/cliconfig_test.go b/command/cliconfig/cliconfig_test.go index 194296c4c..8ae79eb23 100644 --- a/command/cliconfig/cliconfig_test.go +++ b/command/cliconfig/cliconfig_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/davecgh/go-spew/spew" + "github.com/google/go-cmp/cmp" ) // This is the directory where our test fixtures are. @@ -169,6 +170,29 @@ func TestConfigValidate(t *testing.T) { }, 1, // no more than one credentials_helper block allowed }, + "provider_installation good none": { + &Config{ + ProviderInstallation: nil, + }, + 0, + }, + "provider_installation good one": { + &Config{ + ProviderInstallation: []*ProviderInstallation{ + {}, + }, + }, + 0, + }, + "provider_installation too many": { + &Config{ + ProviderInstallation: []*ProviderInstallation{ + {}, + {}, + }, + }, + 1, // no more than one provider_installation block allowed + }, } for name, test := range tests { @@ -209,6 +233,19 @@ func TestConfig_Merge(t *testing.T) { CredentialsHelpers: map[string]*ConfigCredentialsHelper{ "buz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("a")}, + {Location: ProviderInstallationFilesystemMirror("b")}, + }, + }, + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("c")}, + }, + }, + }, } c2 := &Config{ @@ -234,6 +271,13 @@ func TestConfig_Merge(t *testing.T) { CredentialsHelpers: map[string]*ConfigCredentialsHelper{ "biz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("d")}, + }, + }, + }, } expected := &Config{ @@ -270,11 +314,29 @@ func TestConfig_Merge(t *testing.T) { "buz": {}, "biz": {}, }, + ProviderInstallation: []*ProviderInstallation{ + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("a")}, + {Location: ProviderInstallationFilesystemMirror("b")}, + }, + }, + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("c")}, + }, + }, + { + Sources: []*ProviderInstallationSource{ + {Location: ProviderInstallationFilesystemMirror("d")}, + }, + }, + }, } actual := c1.Merge(c2) - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("bad: %#v", actual) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Fatalf("wrong result\n%s", diff) } } diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go new file mode 100644 index 000000000..10e8f4fc1 --- /dev/null +++ b/command/cliconfig/provider_installation.go @@ -0,0 +1,236 @@ +package cliconfig + +import ( + "fmt" + + "github.com/hashicorp/hcl" + hclast "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/terraform/tfdiags" +) + +// ProviderInstallation is the structure of the "provider_installation" +// nested block within the CLI configuration. +type ProviderInstallation struct { + Sources []*ProviderInstallationSource +} + +// decodeProviderInstallationFromConfig uses the HCL AST API directly to +// decode "provider_installation" blocks from the given file. +// +// This uses the HCL AST directly, rather than HCL's decoder, because the +// intended configuration structure can't be represented using the HCL +// decoder's struct tags. This structure is intended as something that would +// be relatively easier to deal with in HCL 2 once we eventually migrate +// CLI config over to that, and so this function is stricter than HCL 1's +// decoder would be in terms of exactly what configuration shape it is +// expecting. +// +// Note that this function wants the top-level file object which might or +// might not contain provider_installation blocks, not a provider_installation +// block directly itself. +func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInstallation, tfdiags.Diagnostics) { + var ret []*ProviderInstallation + var diags tfdiags.Diagnostics + + root := hclFile.Node.(*hclast.ObjectList) + + // This is a rather odd hybrid: it's a HCL 2-like decode implemented using + // the HCL 1 AST API. That makes it a bit awkward in places, but it allows + // us to mimick the strictness of HCL 2 (making a later migration easier) + // and to support a block structure that the HCL 1 decoder can't represent. + for _, block := range root.Items { + if block.Keys[0].Token.Value() != "provider_installation" { + continue + } + if block.Assign.Line != 0 { + // Seems to be an attribute rather than a block + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()), + )) + continue + } + if len(block.Keys) > 1 { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not have any labels.", block.Pos()), + )) + } + + pi := &ProviderInstallation{} + + // Because we checked block.Assign was unset above we can assume that + // we're reading something produced with block syntax and therefore + // it will always be an hclast.ObjectType. + body := block.Val.(*hclast.ObjectType) + + for _, sourceBlock := range body.List.Items { + if sourceBlock.Assign.Line != 0 { + // Seems to be an attribute rather than a block + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()), + )) + continue + } + if len(sourceBlock.Keys) > 1 { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("The blocks inside the provider_installation block at %s may not have any labels.", block.Pos()), + )) + } + + sourceBody := sourceBlock.Val.(*hclast.ObjectType) + + sourceTypeStr := sourceBlock.Keys[0].Token.Value().(string) + var location ProviderInstallationSourceLocation + var include, exclude []string + var extraArgs []string + switch sourceTypeStr { + case "direct": + type BodyContent struct { + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, sourceBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + )) + continue + } + location = ProviderInstallationDirect + include = bodyContent.Include + exclude = bodyContent.Exclude + case "filesystem_mirror": + type BodyContent struct { + Path string `hcl:"path"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, sourceBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + )) + continue + } + if bodyContent.Path == "" { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: \"path\" argument is required.", sourceTypeStr, block.Pos()), + )) + continue + } + location = ProviderInstallationFilesystemMirror(bodyContent.Path) + include = bodyContent.Include + exclude = bodyContent.Exclude + case "network_mirror": + type BodyContent struct { + Host string `hcl:"host"` + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` + } + var bodyContent BodyContent + err := hcl.DecodeObject(&bodyContent, sourceBody) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + )) + continue + } + if bodyContent.Host == "" { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: \"host\" argument is required.", sourceTypeStr, block.Pos()), + )) + continue + } + location = ProviderInstallationNetworkMirror(bodyContent.Host) + include = bodyContent.Include + exclude = bodyContent.Exclude + default: + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Unknown provider installation source type %q at %s.", sourceTypeStr, sourceBlock.Pos()), + )) + continue + } + + for _, argName := range extraArgs { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation source block", + fmt.Sprintf("Invalid %s block at %s: this source type does not expect the argument %q.", sourceTypeStr, block.Pos(), argName), + )) + } + + pi.Sources = append(pi.Sources, &ProviderInstallationSource{ + Location: location, + Include: include, + Exclude: exclude, + }) + } + + ret = append(ret, pi) + } + + return ret, diags +} + +// ProviderInstallationSource represents an installation source block inside +// a provider_installation block. +type ProviderInstallationSource struct { + Location ProviderInstallationSourceLocation + Include []string `hcl:"include"` + Exclude []string `hcl:"exclude"` +} + +// ProviderInstallationSourceLocation is an interface type representing the +// different installation source types. The concrete implementations of +// this interface are: +// +// ProviderInstallationDirect: install from the provider's origin registry +// ProviderInstallationFilesystemMirror(dir): install from a local filesystem mirror +// ProviderInstallationNetworkMirror(host): install from a network mirror +type ProviderInstallationSourceLocation interface { + providerInstallationLocation() +} + +type configProviderInstallationDirect [0]byte + +func (i configProviderInstallationDirect) providerInstallationLocation() {} + +// ProviderInstallationDirect is a ProviderInstallationSourceLocation +// representing installation from a provider's origin registry. +var ProviderInstallationDirect ProviderInstallationSourceLocation = configProviderInstallationDirect{} + +// ProviderInstallationFilesystemMirror is a ProviderInstallationSourceLocation +// representing installation from a particular local filesystem mirror. The +// string value is the filesystem path to the mirror directory. +type ProviderInstallationFilesystemMirror string + +func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {} + +// ProviderInstallationNetworkMirror is a ProviderInstallationSourceLocation +// representing installation from a particular local network mirror. The +// string value is the hostname exactly as written in the configuration, without +// any normalization. +type ProviderInstallationNetworkMirror string + +func (i ProviderInstallationNetworkMirror) providerInstallationLocation() {} diff --git a/command/cliconfig/provider_installation_test.go b/command/cliconfig/provider_installation_test.go new file mode 100644 index 000000000..a9cb7e00c --- /dev/null +++ b/command/cliconfig/provider_installation_test.go @@ -0,0 +1,66 @@ +package cliconfig + +import ( + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestLoadConfig_providerInstallation(t *testing.T) { + got, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation")) + if diags.HasErrors() { + t.Errorf("unexpected diagnostics: %s", diags.Err().Error()) + } + + want := &Config{ + ProviderInstallation: []*ProviderInstallation{ + { + Sources: []*ProviderInstallationSource{ + { + Location: ProviderInstallationFilesystemMirror("/tmp/example1"), + Include: []string{"example.com/*/*"}, + }, + { + Location: ProviderInstallationNetworkMirror("tf-Mirror.example.com"), + Include: []string{"registry.terraform.io/*/*"}, + Exclude: []string{"registry.Terraform.io/foobar/*"}, + }, + { + Location: ProviderInstallationFilesystemMirror("/tmp/example2"), + }, + { + Location: ProviderInstallationDirect, + Exclude: []string{"example.com/*/*"}, + }, + }, + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } +} + +func TestLoadConfig_providerInstallationErrors(t *testing.T) { + _, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation-errors")) + want := `7 problems: + +- Invalid provider_installation source block: Unknown provider installation source type "not_a_thing" at 2:3. +- Invalid provider_installation source block: Invalid filesystem_mirror block at 1:1: "path" argument is required. +- Invalid provider_installation source block: Invalid network_mirror block at 1:1: "host" argument is required. +- Invalid provider_installation source block: The items inside the provider_installation block at 1:1 must all be blocks. +- Invalid provider_installation source block: The blocks inside the provider_installation block at 1:1 may not have any labels. +- Invalid provider_installation block: The provider_installation block at 9:1 must not have any labels. +- Invalid provider_installation block: The provider_installation block at 11:1 must not be introduced with an equals sign.` + + // The above error messages include only line/column location information + // and not file location information because HCL 1 does not store + // information about the filename a location belongs to. (There is a field + // for it in token.Pos but it's always an empty string in practice.) + + if got := diags.Err().Error(); got != want { + t.Errorf("wrong diagnostics\ngot:\n%s\nwant:\n%s", got, want) + } +} diff --git a/command/cliconfig/testdata/provider-installation b/command/cliconfig/testdata/provider-installation new file mode 100644 index 000000000..ff2d5fbec --- /dev/null +++ b/command/cliconfig/testdata/provider-installation @@ -0,0 +1,17 @@ +provider_installation { + filesystem_mirror { + path = "/tmp/example1" + include = ["example.com/*/*"] + } + network_mirror { + host = "tf-Mirror.example.com" + include = ["registry.terraform.io/*/*"] + exclude = ["registry.Terraform.io/foobar/*"] + } + filesystem_mirror { + path = "/tmp/example2" + } + direct { + exclude = ["example.com/*/*"] + } +} diff --git a/command/cliconfig/testdata/provider-installation-errors b/command/cliconfig/testdata/provider-installation-errors new file mode 100644 index 000000000..8cf634e50 --- /dev/null +++ b/command/cliconfig/testdata/provider-installation-errors @@ -0,0 +1,11 @@ +provider_installation { + not_a_thing {} # unknown source type + filesystem_mirror {} # missing "path" argument + network_mirror {} # missing "host" argument + direct = {} # should be a block, not an argument + direct "what" {} # should not have a label +} + +provider_installation "what" {} # should not have a label + +provider_installation = {} # should be a block, not an argument From c5bd783ebaecd3f895c72c079b67e7d61b493182 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Apr 2020 16:26:43 -0700 Subject: [PATCH 02/14] internal/getproviders: Stub NetworkMirrorSource This is a placeholder for later implementation of a mirror source that talks to a particular remote HTTP server and expects it to implement the provider mirror protocol. --- .../getproviders/network_mirror_source.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 internal/getproviders/network_mirror_source.go diff --git a/internal/getproviders/network_mirror_source.go b/internal/getproviders/network_mirror_source.go new file mode 100644 index 000000000..26b8ba0f8 --- /dev/null +++ b/internal/getproviders/network_mirror_source.go @@ -0,0 +1,39 @@ +package getproviders + +import ( + "fmt" + + svchost "github.com/hashicorp/terraform-svchost" + + "github.com/hashicorp/terraform/addrs" +) + +// NetworkMirrorSource is a source that reads providers and their metadata +// from an HTTP server implementing the Terraform network mirror protocol. +type NetworkMirrorSource struct { + host svchost.Hostname +} + +var _ Source = (*NetworkMirrorSource)(nil) + +// NewNetworkMirrorSource constructs and returns a new network-based +// mirror source that will expect to find a mirror service on the given +// host. +func NewNetworkMirrorSource(host svchost.Hostname) *NetworkMirrorSource { + return &NetworkMirrorSource{ + host: host, + } +} + +// AvailableVersions retrieves the available versions for the given provider +// from the network mirror. +func (s *NetworkMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) { + return nil, fmt.Errorf("Network provider mirror is not supported in this version of Terraform") +} + +// PackageMeta checks to see if the network mirror contains a copy of the +// distribution package for the given provider version on the given target, +// and returns the metadata about it if so. +func (s *NetworkMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { + return PackageMeta{}, fmt.Errorf("Network provider mirror is not supported in this version of Terraform") +} From 5af1e6234ab6da412fb8637393c5a17a1b293663 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Apr 2020 16:28:59 -0700 Subject: [PATCH 03/14] main: Honor explicit provider_installation CLI config when present If the CLI configuration contains a provider_installation block then we'll use the source configuration it describes instead of the implied one we'd build otherwise. --- main.go | 18 ++++++++- provider_source.go | 93 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index f9a3b9cf8..09da719c4 100644 --- a/main.go +++ b/main.go @@ -127,6 +127,7 @@ func wrappedMain() int { log.Printf("[INFO] CLI args: %#v", os.Args) config, diags := cliconfig.LoadConfig() + if len(diags) > 0 { // Since we haven't instantiated a command.Meta yet, we need to do // some things manually here and use some "safe" defaults for things @@ -168,7 +169,22 @@ func wrappedMain() int { // direct from a registry. In future there should be a mechanism to // configure providers sources from the CLI config, which will then // change how we construct this object. - providerSrc := providerSource(services) + providerSrc, diags := providerSource(config.ProviderInstallation, services) + if len(diags) > 0 { + Ui.Error("There are some problems with the provider_installation configuration:") + for _, diag := range diags { + earlyColor := &colorstring.Colorize{ + Colors: colorstring.DefaultColors, + Disable: true, // Disable color to be conservative until we know better + Reset: true, + } + Ui.Error(format.Diagnostic(diag, nil, earlyColor, 78)) + } + if diags.HasErrors() { + Ui.Error("As a result of the above problems, Terraform's provider installer may not behave as intended.\n\n") + // We continue to run anyway, because most commands don't do provider installation. + } + } // Initialize the backends. backendInit.Init(services) diff --git a/provider_source.go b/provider_source.go index 8e91658a8..bfbdf0f3f 100644 --- a/provider_source.go +++ b/provider_source.go @@ -1,28 +1,76 @@ package main import ( + "fmt" "log" "os" "path/filepath" "github.com/apparentlymart/go-userdirs/userdirs" + svchost "github.com/hashicorp/terraform-svchost" "github.com/hashicorp/terraform-svchost/disco" "github.com/hashicorp/terraform/addrs" "github.com/hashicorp/terraform/command/cliconfig" "github.com/hashicorp/terraform/internal/getproviders" + "github.com/hashicorp/terraform/tfdiags" ) // providerSource constructs a provider source based on a combination of the // CLI configuration and some default search locations. This will be the // provider source used for provider installation in the "terraform init" // command, unless overridden by the special -plugin-dir option. -func providerSource(services *disco.Disco) getproviders.Source { - // We're not yet using the CLI config here because we've not implemented - // yet the new configuration constructs to customize provider search - // locations. That'll come later. For now, we just always use the - // implicit default provider source. - return implicitProviderSource(services) +func providerSource(configs []*cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + if len(configs) == 0 { + // If there's no explicit installation configuration then we'll build + // up an implicit one with direct registry installation along with + // some automatically-selected local filesystem mirrors. + return implicitProviderSource(services), nil + } + + // There should only be zero or one configurations, which is checked by + // the validation logic in the cliconfig package. Therefore we'll just + // ignore any additional configurations in here. + config := configs[0] + return explicitProviderSource(config, services) +} + +func explicitProviderSource(config *cliconfig.ProviderInstallation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + var searchRules []getproviders.MultiSourceSelector + + for _, sourceConfig := range config.Sources { + source, moreDiags := providerSourceForCLIConfigLocation(sourceConfig.Location, services) + diags = diags.Append(moreDiags) + if moreDiags.HasErrors() { + continue + } + + include, err := getproviders.ParseMultiSourceMatchingPatterns(sourceConfig.Include) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider source inclusion patterns", + fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err), + )) + } + exclude, err := getproviders.ParseMultiSourceMatchingPatterns(sourceConfig.Include) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider source exclusion patterns", + fmt.Sprintf("CLI config specifies invalid provider exclusion patterns: %s.", err), + )) + } + + searchRules = append(searchRules, getproviders.MultiSourceSelector{ + Source: source, + Include: include, + Exclude: exclude, + }) + } + + return getproviders.MultiSource(searchRules), diags } // implicitProviderSource builds a default provider source to use if there's @@ -130,3 +178,36 @@ func implicitProviderSource(services *disco.Disco) getproviders.Source { return getproviders.MultiSource(searchRules) } + +func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationSourceLocation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { + if loc == cliconfig.ProviderInstallationDirect { + return getproviders.NewMemoizeSource( + getproviders.NewRegistrySource(services), + ), nil + } + + switch loc := loc.(type) { + + case cliconfig.ProviderInstallationFilesystemMirror: + return getproviders.NewFilesystemMirrorSource(string(loc)), nil + + case cliconfig.ProviderInstallationNetworkMirror: + host, err := svchost.ForComparison(string(loc)) + if err != nil { + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid hostname for provider installation source", + fmt.Sprintf("Cannot parse %q as a hostname for a network provider mirror: %s.", string(loc), err), + )) + return nil, diags + } + return getproviders.NewNetworkMirrorSource(host), nil + + default: + // We should not get here because the set of cases above should + // be comprehensive for all of the + // cliconfig.ProviderInstallationLocation implementations. + panic(fmt.Sprintf("unexpected provider source location type %T", loc)) + } +} From 94b87e056b086e10606b0402f547fa61cf75c037 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 23 Apr 2020 10:49:19 -0700 Subject: [PATCH 04/14] fixup main.go comment about providersource --- main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main.go b/main.go index 09da719c4..c11432c3f 100644 --- a/main.go +++ b/main.go @@ -165,10 +165,6 @@ func wrappedMain() int { services := disco.NewWithCredentialsSource(credsSrc) services.SetUserAgent(httpclient.TerraformUserAgent(version.String())) - // For the moment, we just always use the registry source to install - // direct from a registry. In future there should be a mechanism to - // configure providers sources from the CLI config, which will then - // change how we construct this object. providerSrc, diags := providerSource(config.ProviderInstallation, services) if len(diags) > 0 { Ui.Error("There are some problems with the provider_installation configuration:") From b8856c677ca432a1f714d0767b348113aaca31d8 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 21 Apr 2020 16:36:06 -0700 Subject: [PATCH 05/14] cliconfig: Ignore config dir if TF_CLI_CONFIG_FILE envvar is set When we originally introduced this environment variable it was intended to solve for the use-case where a particular invocation of Terraform needs a different CLI configuration than usual, such as if Terraform is being run as part of an automated test suite or other sort of automated situation with different needs than normal use. However, we accidentally had it only override the original singleton CLI config file, while leaving the CLI configuration directory still enabled. Now we'll take the CLI configuration out of the equation too, so that only the single specified configuration file and any other environment-sourced settings will be included. --- command/cliconfig/cliconfig.go | 35 ++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/command/cliconfig/cliconfig.go b/command/cliconfig/cliconfig.go index e36b11cc5..c765132d8 100644 --- a/command/cliconfig/cliconfig.go +++ b/command/cliconfig/cliconfig.go @@ -113,12 +113,23 @@ func LoadConfig() (*Config, tfdiags.Diagnostics) { } } - if configDir, err := ConfigDir(); err == nil { - if info, err := os.Stat(configDir); err == nil && info.IsDir() { - dirConfig, dirDiags := loadConfigDir(configDir) - diags = diags.Append(dirDiags) - config = config.Merge(dirConfig) + // Unless the user has specifically overridden the configuration file + // location using an environment variable, we'll also load what we find + // in the config directory. We skip the config directory when source + // file override is set because we interpret the environment variable + // being set as an intention to ignore the default set of CLI config + // files because we're doing something special, like running Terraform + // in automation with a locally-customized configuration. + if cliConfigFileOverride() == "" { + if configDir, err := ConfigDir(); err == nil { + if info, err := os.Stat(configDir); err == nil && info.IsDir() { + dirConfig, dirDiags := loadConfigDir(configDir) + diags = diags.Append(dirDiags) + config = config.Merge(dirConfig) + } } + } else { + log.Printf("[DEBUG] Not reading CLI config directory because config location is overridden by environment variable") } if envConfig := EnvConfig(); envConfig != nil { @@ -357,11 +368,7 @@ func (c1 *Config) Merge(c2 *Config) *Config { func cliConfigFile() (string, error) { mustExist := true - configFilePath := os.Getenv("TF_CLI_CONFIG_FILE") - if configFilePath == "" { - configFilePath = os.Getenv("TERRAFORM_CONFIG") - } - + configFilePath := cliConfigFileOverride() if configFilePath == "" { var err error configFilePath, err = ConfigFile() @@ -388,3 +395,11 @@ func cliConfigFile() (string, error) { log.Println("[DEBUG] File doesn't exist, but doesn't need to. Ignoring.") return "", nil } + +func cliConfigFileOverride() string { + configFilePath := os.Getenv("TF_CLI_CONFIG_FILE") + if configFilePath == "" { + configFilePath = os.Getenv("TERRAFORM_CONFIG") + } + return configFilePath +} From 8b75d1498f2ccc260d55561d95a921ac11fd885f Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 15:58:40 -0700 Subject: [PATCH 06/14] command/cliconfig: Use existing HTTP mirror source rather than new stub An earlier commit added a redundant stub for a new network mirror source that was already previously stubbed as HTTPMirrorSource. This commit removes the unnecessary extra stub and changes the CLI config handling to use it instead. Along the way this also switches to using a full base URL rather than just a hostname for the mirror, because using the usual "Terraform-native service discovery" protocol here doesn't isn't as useful as in the places we normally use it (the mirror mechanism is already serving as an indirection over the registry protocol) and using a direct base URL will make it easier to deploy an HTTP mirror under a path prefix on an existing static file server. --- command/cliconfig/provider_installation.go | 12 +++--- .../cliconfig/provider_installation_test.go | 4 +- .../cliconfig/testdata/provider-installation | 2 +- internal/getproviders/http_mirror_source.go | 7 ++-- .../getproviders/network_mirror_source.go | 39 ------------------- provider_source.go | 19 ++++++--- 6 files changed, 26 insertions(+), 57 deletions(-) delete mode 100644 internal/getproviders/network_mirror_source.go diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go index 10e8f4fc1..d5e57ac05 100644 --- a/command/cliconfig/provider_installation.go +++ b/command/cliconfig/provider_installation.go @@ -138,7 +138,7 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst exclude = bodyContent.Exclude case "network_mirror": type BodyContent struct { - Host string `hcl:"host"` + URL string `hcl:"url"` Include []string `hcl:"include"` Exclude []string `hcl:"exclude"` } @@ -152,15 +152,15 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst )) continue } - if bodyContent.Host == "" { + if bodyContent.URL == "" { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: \"host\" argument is required.", sourceTypeStr, block.Pos()), + fmt.Sprintf("Invalid %s block at %s: \"url\" argument is required.", sourceTypeStr, block.Pos()), )) continue } - location = ProviderInstallationNetworkMirror(bodyContent.Host) + location = ProviderInstallationNetworkMirror(bodyContent.URL) include = bodyContent.Include exclude = bodyContent.Exclude default: @@ -229,8 +229,8 @@ func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {} // ProviderInstallationNetworkMirror is a ProviderInstallationSourceLocation // representing installation from a particular local network mirror. The -// string value is the hostname exactly as written in the configuration, without -// any normalization. +// string value is the HTTP base URL exactly as written in the configuration, +// without any normalization. type ProviderInstallationNetworkMirror string func (i ProviderInstallationNetworkMirror) providerInstallationLocation() {} diff --git a/command/cliconfig/provider_installation_test.go b/command/cliconfig/provider_installation_test.go index a9cb7e00c..553d0bc75 100644 --- a/command/cliconfig/provider_installation_test.go +++ b/command/cliconfig/provider_installation_test.go @@ -22,7 +22,7 @@ func TestLoadConfig_providerInstallation(t *testing.T) { Include: []string{"example.com/*/*"}, }, { - Location: ProviderInstallationNetworkMirror("tf-Mirror.example.com"), + Location: ProviderInstallationNetworkMirror("https://tf-Mirror.example.com/"), Include: []string{"registry.terraform.io/*/*"}, Exclude: []string{"registry.Terraform.io/foobar/*"}, }, @@ -49,7 +49,7 @@ func TestLoadConfig_providerInstallationErrors(t *testing.T) { - Invalid provider_installation source block: Unknown provider installation source type "not_a_thing" at 2:3. - Invalid provider_installation source block: Invalid filesystem_mirror block at 1:1: "path" argument is required. -- Invalid provider_installation source block: Invalid network_mirror block at 1:1: "host" argument is required. +- Invalid provider_installation source block: Invalid network_mirror block at 1:1: "url" argument is required. - Invalid provider_installation source block: The items inside the provider_installation block at 1:1 must all be blocks. - Invalid provider_installation source block: The blocks inside the provider_installation block at 1:1 may not have any labels. - Invalid provider_installation block: The provider_installation block at 9:1 must not have any labels. diff --git a/command/cliconfig/testdata/provider-installation b/command/cliconfig/testdata/provider-installation index ff2d5fbec..91dc82eab 100644 --- a/command/cliconfig/testdata/provider-installation +++ b/command/cliconfig/testdata/provider-installation @@ -4,7 +4,7 @@ provider_installation { include = ["example.com/*/*"] } network_mirror { - host = "tf-Mirror.example.com" + url = "https://tf-Mirror.example.com/" include = ["registry.terraform.io/*/*"] exclude = ["registry.Terraform.io/foobar/*"] } diff --git a/internal/getproviders/http_mirror_source.go b/internal/getproviders/http_mirror_source.go index 5b5bb80f1..c2dedf9db 100644 --- a/internal/getproviders/http_mirror_source.go +++ b/internal/getproviders/http_mirror_source.go @@ -1,6 +1,7 @@ package getproviders import ( + "fmt" "net/url" "github.com/hashicorp/terraform/addrs" @@ -26,13 +27,11 @@ func NewHTTPMirrorSource(baseURL *url.URL) *HTTPMirrorSource { // AvailableVersions retrieves the available versions for the given provider // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) { - // TODO: Implement - panic("HTTPMirrorSource.AvailableVersions not yet implemented") + return nil, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") } // PackageMeta retrieves metadata for the requested provider package // from the object's underlying HTTP mirror service. func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { - // TODO: Implement - panic("HTTPMirrorSource.PackageMeta not yet implemented") + return PackageMeta{}, fmt.Errorf("Network-based provider mirrors are not supported in this version of Terraform") } diff --git a/internal/getproviders/network_mirror_source.go b/internal/getproviders/network_mirror_source.go deleted file mode 100644 index 26b8ba0f8..000000000 --- a/internal/getproviders/network_mirror_source.go +++ /dev/null @@ -1,39 +0,0 @@ -package getproviders - -import ( - "fmt" - - svchost "github.com/hashicorp/terraform-svchost" - - "github.com/hashicorp/terraform/addrs" -) - -// NetworkMirrorSource is a source that reads providers and their metadata -// from an HTTP server implementing the Terraform network mirror protocol. -type NetworkMirrorSource struct { - host svchost.Hostname -} - -var _ Source = (*NetworkMirrorSource)(nil) - -// NewNetworkMirrorSource constructs and returns a new network-based -// mirror source that will expect to find a mirror service on the given -// host. -func NewNetworkMirrorSource(host svchost.Hostname) *NetworkMirrorSource { - return &NetworkMirrorSource{ - host: host, - } -} - -// AvailableVersions retrieves the available versions for the given provider -// from the network mirror. -func (s *NetworkMirrorSource) AvailableVersions(provider addrs.Provider) (VersionList, error) { - return nil, fmt.Errorf("Network provider mirror is not supported in this version of Terraform") -} - -// PackageMeta checks to see if the network mirror contains a copy of the -// distribution package for the given provider version on the given target, -// and returns the metadata about it if so. -func (s *NetworkMirrorSource) PackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, error) { - return PackageMeta{}, fmt.Errorf("Network provider mirror is not supported in this version of Terraform") -} diff --git a/provider_source.go b/provider_source.go index bfbdf0f3f..1d804342e 100644 --- a/provider_source.go +++ b/provider_source.go @@ -3,11 +3,11 @@ package main import ( "fmt" "log" + "net/url" "os" "path/filepath" "github.com/apparentlymart/go-userdirs/userdirs" - svchost "github.com/hashicorp/terraform-svchost" "github.com/hashicorp/terraform-svchost/disco" "github.com/hashicorp/terraform/addrs" @@ -192,17 +192,26 @@ func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationSource return getproviders.NewFilesystemMirrorSource(string(loc)), nil case cliconfig.ProviderInstallationNetworkMirror: - host, err := svchost.ForComparison(string(loc)) + url, err := url.Parse(string(loc)) if err != nil { var diags tfdiags.Diagnostics diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid hostname for provider installation source", - fmt.Sprintf("Cannot parse %q as a hostname for a network provider mirror: %s.", string(loc), err), + "Invalid URL for provider installation source", + fmt.Sprintf("Cannot parse %q as a URL for a network provider mirror: %s.", string(loc), err), )) return nil, diags } - return getproviders.NewNetworkMirrorSource(host), nil + if url.Scheme != "https" || url.Host == "" { + var diags tfdiags.Diagnostics + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid URL for provider installation source", + fmt.Sprintf("Cannot use %q as a URL for a network provider mirror: the mirror must be at an https: URL.", string(loc)), + )) + return nil, diags + } + return getproviders.NewHTTPMirrorSource(url), nil default: // We should not get here because the set of cases above should From e872ec44618c286691da7f8b6c6f779a1c907800 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 16:08:37 -0700 Subject: [PATCH 07/14] command/cliconfig: Remove remnant extraArg checks in provider_installation In the first pass of implementing this it was strict about what arguments are allowed inside source blocks, but that was counter to our usual design principles for CLI config where we tend to ignore unrecognized things to allow for some limited kinds of future expansion without breaking compatibility with older versions of Terraform that will be sharing the same CLI configuration files with newer versions. However, I'd removed the tracking of that prior to the initial commit. I missed some leftover parts when doing that removal, so this cleans up the rest of it. --- command/cliconfig/provider_installation.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go index d5e57ac05..cca208222 100644 --- a/command/cliconfig/provider_installation.go +++ b/command/cliconfig/provider_installation.go @@ -89,7 +89,6 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst sourceTypeStr := sourceBlock.Keys[0].Token.Value().(string) var location ProviderInstallationSourceLocation var include, exclude []string - var extraArgs []string switch sourceTypeStr { case "direct": type BodyContent struct { @@ -172,14 +171,6 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst continue } - for _, argName := range extraArgs { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: this source type does not expect the argument %q.", sourceTypeStr, block.Pos(), argName), - )) - } - pi.Sources = append(pi.Sources, &ProviderInstallationSource{ Location: location, Include: include, From f5012c12dabf9f9b08d3ea75d3a349674d3d7c7f Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 16:28:06 -0700 Subject: [PATCH 08/14] command/cliconfig: Installation methods, not installation sources Unfortunately in the user model the noun "source" is already used for the argument in the required_providers block to specify which provider to use, so it's confusing to use the same noun to also refer to the method used to obtain that provider. In the hope of mitigating that confusion, here we use the noun "method", as in "installation method", to talk about the decision between getting a provider directly from its origin registry or getting it from some mirror. This is distinct from the provider's "source", which is the location where a provider _originates_ (prior to mirroring). This noun is also not super awesome, but better than overloading an existing term in the same feature. --- command/cliconfig/cliconfig_test.go | 12 ++-- command/cliconfig/provider_installation.go | 66 +++++++++---------- .../cliconfig/provider_installation_test.go | 12 ++-- provider_source.go | 10 +-- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/command/cliconfig/cliconfig_test.go b/command/cliconfig/cliconfig_test.go index 8ae79eb23..a84e825b3 100644 --- a/command/cliconfig/cliconfig_test.go +++ b/command/cliconfig/cliconfig_test.go @@ -235,13 +235,13 @@ func TestConfig_Merge(t *testing.T) { }, ProviderInstallation: []*ProviderInstallation{ { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("a")}, {Location: ProviderInstallationFilesystemMirror("b")}, }, }, { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("c")}, }, }, @@ -273,7 +273,7 @@ func TestConfig_Merge(t *testing.T) { }, ProviderInstallation: []*ProviderInstallation{ { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("d")}, }, }, @@ -316,18 +316,18 @@ func TestConfig_Merge(t *testing.T) { }, ProviderInstallation: []*ProviderInstallation{ { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("a")}, {Location: ProviderInstallationFilesystemMirror("b")}, }, }, { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("c")}, }, }, { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ {Location: ProviderInstallationFilesystemMirror("d")}, }, }, diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go index cca208222..28ca74b2c 100644 --- a/command/cliconfig/provider_installation.go +++ b/command/cliconfig/provider_installation.go @@ -11,7 +11,7 @@ import ( // ProviderInstallation is the structure of the "provider_installation" // nested block within the CLI configuration. type ProviderInstallation struct { - Sources []*ProviderInstallationSource + Methods []*ProviderInstallationMethod } // decodeProviderInstallationFromConfig uses the HCL AST API directly to @@ -66,42 +66,42 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst // it will always be an hclast.ObjectType. body := block.Val.(*hclast.ObjectType) - for _, sourceBlock := range body.List.Items { - if sourceBlock.Assign.Line != 0 { + for _, methodBlock := range body.List.Items { + if methodBlock.Assign.Line != 0 { // Seems to be an attribute rather than a block diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", + "Invalid provider_installation method block", fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()), )) continue } - if len(sourceBlock.Keys) > 1 { + if len(methodBlock.Keys) > 1 { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", + "Invalid provider_installation method block", fmt.Sprintf("The blocks inside the provider_installation block at %s may not have any labels.", block.Pos()), )) } - sourceBody := sourceBlock.Val.(*hclast.ObjectType) + methodBody := methodBlock.Val.(*hclast.ObjectType) - sourceTypeStr := sourceBlock.Keys[0].Token.Value().(string) - var location ProviderInstallationSourceLocation + methodTypeStr := methodBlock.Keys[0].Token.Value().(string) + var location ProviderInstallationLocation var include, exclude []string - switch sourceTypeStr { + switch methodTypeStr { case "direct": type BodyContent struct { Include []string `hcl:"include"` Exclude []string `hcl:"exclude"` } var bodyContent BodyContent - err := hcl.DecodeObject(&bodyContent, sourceBody) + err := hcl.DecodeObject(&bodyContent, methodBody) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), )) continue } @@ -115,20 +115,20 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst Exclude []string `hcl:"exclude"` } var bodyContent BodyContent - err := hcl.DecodeObject(&bodyContent, sourceBody) + err := hcl.DecodeObject(&bodyContent, methodBody) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), )) continue } if bodyContent.Path == "" { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: \"path\" argument is required.", sourceTypeStr, block.Pos()), + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: \"path\" argument is required.", methodTypeStr, block.Pos()), )) continue } @@ -142,20 +142,20 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst Exclude []string `hcl:"exclude"` } var bodyContent BodyContent - err := hcl.DecodeObject(&bodyContent, sourceBody) + err := hcl.DecodeObject(&bodyContent, methodBody) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: %s.", sourceTypeStr, block.Pos(), err), + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: %s.", methodTypeStr, block.Pos(), err), )) continue } if bodyContent.URL == "" { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Invalid %s block at %s: \"url\" argument is required.", sourceTypeStr, block.Pos()), + "Invalid provider_installation method block", + fmt.Sprintf("Invalid %s block at %s: \"url\" argument is required.", methodTypeStr, block.Pos()), )) continue } @@ -165,13 +165,13 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst default: diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, - "Invalid provider_installation source block", - fmt.Sprintf("Unknown provider installation source type %q at %s.", sourceTypeStr, sourceBlock.Pos()), + "Invalid provider_installation method block", + fmt.Sprintf("Unknown provider installation method %q at %s.", methodTypeStr, methodBlock.Pos()), )) continue } - pi.Sources = append(pi.Sources, &ProviderInstallationSource{ + pi.Methods = append(pi.Methods, &ProviderInstallationMethod{ Location: location, Include: include, Exclude: exclude, @@ -184,22 +184,22 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst return ret, diags } -// ProviderInstallationSource represents an installation source block inside +// ProviderInstallationMethod represents an installation method block inside // a provider_installation block. -type ProviderInstallationSource struct { - Location ProviderInstallationSourceLocation +type ProviderInstallationMethod struct { + Location ProviderInstallationLocation Include []string `hcl:"include"` Exclude []string `hcl:"exclude"` } -// ProviderInstallationSourceLocation is an interface type representing the -// different installation source types. The concrete implementations of +// ProviderInstallationLocation is an interface type representing the +// different installation location types. The concrete implementations of // this interface are: // // ProviderInstallationDirect: install from the provider's origin registry // ProviderInstallationFilesystemMirror(dir): install from a local filesystem mirror // ProviderInstallationNetworkMirror(host): install from a network mirror -type ProviderInstallationSourceLocation interface { +type ProviderInstallationLocation interface { providerInstallationLocation() } @@ -209,7 +209,7 @@ func (i configProviderInstallationDirect) providerInstallationLocation() {} // ProviderInstallationDirect is a ProviderInstallationSourceLocation // representing installation from a provider's origin registry. -var ProviderInstallationDirect ProviderInstallationSourceLocation = configProviderInstallationDirect{} +var ProviderInstallationDirect ProviderInstallationLocation = configProviderInstallationDirect{} // ProviderInstallationFilesystemMirror is a ProviderInstallationSourceLocation // representing installation from a particular local filesystem mirror. The diff --git a/command/cliconfig/provider_installation_test.go b/command/cliconfig/provider_installation_test.go index 553d0bc75..0712e0e9f 100644 --- a/command/cliconfig/provider_installation_test.go +++ b/command/cliconfig/provider_installation_test.go @@ -16,7 +16,7 @@ func TestLoadConfig_providerInstallation(t *testing.T) { want := &Config{ ProviderInstallation: []*ProviderInstallation{ { - Sources: []*ProviderInstallationSource{ + Methods: []*ProviderInstallationMethod{ { Location: ProviderInstallationFilesystemMirror("/tmp/example1"), Include: []string{"example.com/*/*"}, @@ -47,11 +47,11 @@ func TestLoadConfig_providerInstallationErrors(t *testing.T) { _, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation-errors")) want := `7 problems: -- Invalid provider_installation source block: Unknown provider installation source type "not_a_thing" at 2:3. -- Invalid provider_installation source block: Invalid filesystem_mirror block at 1:1: "path" argument is required. -- Invalid provider_installation source block: Invalid network_mirror block at 1:1: "url" argument is required. -- Invalid provider_installation source block: The items inside the provider_installation block at 1:1 must all be blocks. -- Invalid provider_installation source block: The blocks inside the provider_installation block at 1:1 may not have any labels. +- Invalid provider_installation method block: Unknown provider installation method "not_a_thing" at 2:3. +- Invalid provider_installation method block: Invalid filesystem_mirror block at 1:1: "path" argument is required. +- Invalid provider_installation method block: Invalid network_mirror block at 1:1: "url" argument is required. +- Invalid provider_installation method block: The items inside the provider_installation block at 1:1 must all be blocks. +- Invalid provider_installation method block: The blocks inside the provider_installation block at 1:1 may not have any labels. - Invalid provider_installation block: The provider_installation block at 9:1 must not have any labels. - Invalid provider_installation block: The provider_installation block at 11:1 must not be introduced with an equals sign.` diff --git a/provider_source.go b/provider_source.go index 1d804342e..bdda08bd3 100644 --- a/provider_source.go +++ b/provider_source.go @@ -39,14 +39,14 @@ func explicitProviderSource(config *cliconfig.ProviderInstallation, services *di var diags tfdiags.Diagnostics var searchRules []getproviders.MultiSourceSelector - for _, sourceConfig := range config.Sources { - source, moreDiags := providerSourceForCLIConfigLocation(sourceConfig.Location, services) + for _, methodConfig := range config.Methods { + source, moreDiags := providerSourceForCLIConfigLocation(methodConfig.Location, services) diags = diags.Append(moreDiags) if moreDiags.HasErrors() { continue } - include, err := getproviders.ParseMultiSourceMatchingPatterns(sourceConfig.Include) + include, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, @@ -54,7 +54,7 @@ func explicitProviderSource(config *cliconfig.ProviderInstallation, services *di fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err), )) } - exclude, err := getproviders.ParseMultiSourceMatchingPatterns(sourceConfig.Include) + exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, @@ -179,7 +179,7 @@ func implicitProviderSource(services *disco.Disco) getproviders.Source { return getproviders.MultiSource(searchRules) } -func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationSourceLocation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { +func providerSourceForCLIConfigLocation(loc cliconfig.ProviderInstallationLocation, services *disco.Disco) (getproviders.Source, tfdiags.Diagnostics) { if loc == cliconfig.ProviderInstallationDirect { return getproviders.NewMemoizeSource( getproviders.NewRegistrySource(services), From c7fe6b9160cbf0038bc254b819561799e743bef7 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 17:12:33 -0700 Subject: [PATCH 09/14] command/cliconfig: handle provider_installation block in JSON syntax The CLI config can be written in both native HCL and HCL JSON syntaxes, so the provider_installation block must be expressible using JSON too. Our previous checks to approximate HCL 2-level strictness were too strict for HCL JSON where things are more ambiguous even in HCL 2, so this includes some additional relaxations if we detect that we're decoding an AST produced from a JSON file. This is still subject to the quirky ways HCL 1 handles JSON though, so the JSON value must be structured in a way that doesn't trigger HCL's heuristics that try to guess what is a block and what is an attribute. (This is the issue that HCL 2 fixes by always decoding using a schema; there's more context on this in: https://log.martinatkins.me/2019/04/25/hcl-json/ ) --- command/cliconfig/provider_installation.go | 44 +++++++++++--- .../cliconfig/provider_installation_test.go | 58 ++++++++++--------- .../testdata/provider-installation.json | 19 ++++++ 3 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 command/cliconfig/testdata/provider-installation.json diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go index 28ca74b2c..3d4919950 100644 --- a/command/cliconfig/provider_installation.go +++ b/command/cliconfig/provider_installation.go @@ -42,7 +42,12 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst if block.Keys[0].Token.Value() != "provider_installation" { continue } - if block.Assign.Line != 0 { + // HCL only tracks whether the input was JSON or native syntax inside + // individual tokens, so we'll use our block type token to decide + // and assume that the rest of the block must be written in the same + // syntax, because syntax is a whole-file idea. + isJSON := block.Keys[0].Token.JSON + if block.Assign.Line != 0 && !isJSON { // Seems to be an attribute rather than a block diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, @@ -51,7 +56,7 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst )) continue } - if len(block.Keys) > 1 { + if len(block.Keys) > 1 && !isJSON { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Invalid provider_installation block", @@ -61,13 +66,22 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst pi := &ProviderInstallation{} - // Because we checked block.Assign was unset above we can assume that - // we're reading something produced with block syntax and therefore - // it will always be an hclast.ObjectType. - body := block.Val.(*hclast.ObjectType) + body, ok := block.Val.(*hclast.ObjectType) + if !ok { + // We can't get in here with native HCL syntax because we + // already checked above that we're using block syntax, but + // if we're reading JSON then our value could potentially be + // anything. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation block", + fmt.Sprintf("The provider_installation block at %s must not be introduced with an equals sign.", block.Pos()), + )) + continue + } for _, methodBlock := range body.List.Items { - if methodBlock.Assign.Line != 0 { + if methodBlock.Assign.Line != 0 && !isJSON { // Seems to be an attribute rather than a block diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, @@ -76,7 +90,7 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst )) continue } - if len(methodBlock.Keys) > 1 { + if len(methodBlock.Keys) > 1 && !isJSON { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Invalid provider_installation method block", @@ -84,7 +98,19 @@ func decodeProviderInstallationFromConfig(hclFile *hclast.File) ([]*ProviderInst )) } - methodBody := methodBlock.Val.(*hclast.ObjectType) + methodBody, ok := methodBlock.Val.(*hclast.ObjectType) + if !ok { + // We can't get in here with native HCL syntax because we + // already checked above that we're using block syntax, but + // if we're reading JSON then our value could potentially be + // anything. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Invalid provider_installation method block", + fmt.Sprintf("The items inside the provider_installation block at %s must all be blocks.", block.Pos()), + )) + continue + } methodTypeStr := methodBlock.Keys[0].Token.Value().(string) var location ProviderInstallationLocation diff --git a/command/cliconfig/provider_installation_test.go b/command/cliconfig/provider_installation_test.go index 0712e0e9f..5eb0b412f 100644 --- a/command/cliconfig/provider_installation_test.go +++ b/command/cliconfig/provider_installation_test.go @@ -8,38 +8,42 @@ import ( ) func TestLoadConfig_providerInstallation(t *testing.T) { - got, diags := loadConfigFile(filepath.Join(fixtureDir, "provider-installation")) - if diags.HasErrors() { - t.Errorf("unexpected diagnostics: %s", diags.Err().Error()) - } + for _, configFile := range []string{"provider-installation", "provider-installation.json"} { + t.Run(configFile, func(t *testing.T) { + got, diags := loadConfigFile(filepath.Join(fixtureDir, configFile)) + if diags.HasErrors() { + t.Errorf("unexpected diagnostics: %s", diags.Err().Error()) + } - want := &Config{ - ProviderInstallation: []*ProviderInstallation{ - { - Methods: []*ProviderInstallationMethod{ + want := &Config{ + ProviderInstallation: []*ProviderInstallation{ { - Location: ProviderInstallationFilesystemMirror("/tmp/example1"), - Include: []string{"example.com/*/*"}, - }, - { - Location: ProviderInstallationNetworkMirror("https://tf-Mirror.example.com/"), - Include: []string{"registry.terraform.io/*/*"}, - Exclude: []string{"registry.Terraform.io/foobar/*"}, - }, - { - Location: ProviderInstallationFilesystemMirror("/tmp/example2"), - }, - { - Location: ProviderInstallationDirect, - Exclude: []string{"example.com/*/*"}, + Methods: []*ProviderInstallationMethod{ + { + Location: ProviderInstallationFilesystemMirror("/tmp/example1"), + Include: []string{"example.com/*/*"}, + }, + { + Location: ProviderInstallationNetworkMirror("https://tf-Mirror.example.com/"), + Include: []string{"registry.terraform.io/*/*"}, + Exclude: []string{"registry.Terraform.io/foobar/*"}, + }, + { + Location: ProviderInstallationFilesystemMirror("/tmp/example2"), + }, + { + Location: ProviderInstallationDirect, + Exclude: []string{"example.com/*/*"}, + }, + }, }, }, - }, - }, - } + } - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("wrong result\n%s", diff) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) } } diff --git a/command/cliconfig/testdata/provider-installation.json b/command/cliconfig/testdata/provider-installation.json new file mode 100644 index 000000000..826d68d47 --- /dev/null +++ b/command/cliconfig/testdata/provider-installation.json @@ -0,0 +1,19 @@ +{ + "provider_installation": { + "filesystem_mirror": [{ + "path": "/tmp/example1", + "include": ["example.com/*/*"] + }], + "network_mirror": [{ + "url": "https://tf-Mirror.example.com/", + "include": ["registry.terraform.io/*/*"], + "exclude": ["registry.Terraform.io/foobar/*"] + }], + "filesystem_mirror": [{ + "path": "/tmp/example2" + }], + "direct": [{ + "exclude": ["example.com/*/*"] + }] + } +} From 3167067029837e545033b02f535f814e360e43f2 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 17:13:36 -0700 Subject: [PATCH 10/14] command/e2etest: provider installation with explicit install methods This exercises the ability to customize the installation methods used by the provider plugin installer, in this case forcing the use of a custom local directory with a result essentially the same as what happens when you pass -plugin-dir to "terraform init". --- command/e2etest/init_test.go | 50 +++++++++++++++++++ .../cliconfig.tfrc | 5 ++ .../cliconfig.tfrc.json | 9 ++++ .../terraform-provider-happycloud_v1.2.0 | 2 + .../custom-provider-install-method/main.tf | 21 ++++++++ 5 files changed, 87 insertions(+) create mode 100644 command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc create mode 100644 command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json create mode 100644 command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 create mode 100644 command/e2etest/testdata/custom-provider-install-method/main.tf diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go index c1ad00805..7993c4e13 100644 --- a/command/e2etest/init_test.go +++ b/command/e2etest/init_test.go @@ -173,6 +173,56 @@ func TestInitProvidersLocalOnly(t *testing.T) { } } +func TestInitProvidersCustomMethod(t *testing.T) { + t.Parallel() + + // This test should not reach out to the network if it is behaving as + // intended. If it _does_ try to access an upstream registry and encounter + // an error doing so then that's a legitimate test failure that should be + // fixed. (If it incorrectly reaches out anywhere then it's likely to be + // to the host "example.com", which is the placeholder domain we use in + // the test fixture.) + + for _, configFile := range []string{"cliconfig.tfrc", "cliconfig.tfrc.json"} { + t.Run(configFile, func(t *testing.T) { + fixturePath := filepath.Join("testdata", "custom-provider-install-method") + tf := e2e.NewBinary(terraformBin, fixturePath) + defer tf.Close() + + // Our fixture dir has a generic os_arch dir, which we need to customize + // to the actual OS/arch where this test is running in order to get the + // desired result. + fixtMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch") + wantMachineDir := tf.Path("fs-mirror/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)) + err := os.Rename(fixtMachineDir, wantMachineDir) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + // We'll use a local CLI configuration file taken from our fixture + // directory so we can force a custom installation method config. + tf.AddEnv("TF_CLI_CONFIG_FILE=" + tf.Path(configFile)) + + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + if !strings.Contains(stdout, "Terraform has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") { + t.Errorf("provider download message is missing from output:\n%s", stdout) + } + }) + } +} + func TestInitProviders_pluginCache(t *testing.T) { t.Parallel() diff --git a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc new file mode 100644 index 000000000..a0c015e23 --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc @@ -0,0 +1,5 @@ +provider_installation { + filesystem_mirror { + path = "./fs-mirror" + } +} diff --git a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json new file mode 100644 index 000000000..6e5e946b8 --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc.json @@ -0,0 +1,9 @@ +{ + "provider_installation": { + "filesystem_mirror": [ + { + "path": "./fs-mirror" + } + ] + } +} diff --git a/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 b/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 new file mode 100644 index 000000000..3299bec8a --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/fs-mirror/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0 @@ -0,0 +1,2 @@ +This is not a real plugin executable. It's just here to be discovered by the +provider installation process. diff --git a/command/e2etest/testdata/custom-provider-install-method/main.tf b/command/e2etest/testdata/custom-provider-install-method/main.tf new file mode 100644 index 000000000..a521cf07b --- /dev/null +++ b/command/e2etest/testdata/custom-provider-install-method/main.tf @@ -0,0 +1,21 @@ +# The purpose of this test is to refer to a provider whose address contains +# a hostname that is only used for namespacing purposes and doesn't actually +# have a provider registry deployed at it. +# +# A user can install such a provider in one of the implied local filesystem +# directories and Terraform should accept that as the selection for that +# provider without producing any errors about the fact that example.com +# does not have a provider registry. +# +# For this test in particular we're using the "vendor" directory that is +# the documented way to include provider plugins directly inside a +# configuration uploaded to Terraform Cloud, but this functionality applies +# to all of the implicit local filesystem search directories. + +terraform { + required_providers { + happycloud = { + source = "example.com/awesomecorp/happycloud" + } + } +} From 6b2050f42a556488c855b1ed71fdc8b7f6684d4d Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 17:31:57 -0700 Subject: [PATCH 11/14] main: Properly handle provider installation method exclusions Previously we were incorrectly using the Include configuration for both the include and exclude list, making the include portion totally ineffective. --- command/cliconfig/provider_installation.go | 18 +++++++++++++++--- .../cliconfig.tfrc | 3 +++ provider_source.go | 7 ++++++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/command/cliconfig/provider_installation.go b/command/cliconfig/provider_installation.go index 3d4919950..6ef8833fc 100644 --- a/command/cliconfig/provider_installation.go +++ b/command/cliconfig/provider_installation.go @@ -229,13 +229,17 @@ type ProviderInstallationLocation interface { providerInstallationLocation() } -type configProviderInstallationDirect [0]byte +type providerInstallationDirect [0]byte -func (i configProviderInstallationDirect) providerInstallationLocation() {} +func (i providerInstallationDirect) providerInstallationLocation() {} // ProviderInstallationDirect is a ProviderInstallationSourceLocation // representing installation from a provider's origin registry. -var ProviderInstallationDirect ProviderInstallationLocation = configProviderInstallationDirect{} +var ProviderInstallationDirect ProviderInstallationLocation = providerInstallationDirect{} + +func (i providerInstallationDirect) GoString() string { + return "cliconfig.ProviderInstallationDirect" +} // ProviderInstallationFilesystemMirror is a ProviderInstallationSourceLocation // representing installation from a particular local filesystem mirror. The @@ -244,6 +248,10 @@ type ProviderInstallationFilesystemMirror string func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {} +func (i ProviderInstallationFilesystemMirror) GoString() string { + return fmt.Sprintf("cliconfig.ProviderInstallationFilesystemMirror(%q)", i) +} + // ProviderInstallationNetworkMirror is a ProviderInstallationSourceLocation // representing installation from a particular local network mirror. The // string value is the HTTP base URL exactly as written in the configuration, @@ -251,3 +259,7 @@ func (i ProviderInstallationFilesystemMirror) providerInstallationLocation() {} type ProviderInstallationNetworkMirror string func (i ProviderInstallationNetworkMirror) providerInstallationLocation() {} + +func (i ProviderInstallationNetworkMirror) GoString() string { + return fmt.Sprintf("cliconfig.ProviderInstallationNetworkMirror(%q)", i) +} diff --git a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc index a0c015e23..4b4dbefa7 100644 --- a/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc +++ b/command/e2etest/testdata/custom-provider-install-method/cliconfig.tfrc @@ -2,4 +2,7 @@ provider_installation { filesystem_mirror { path = "./fs-mirror" } + direct { + exclude = ["example.com/*/*"] + } } diff --git a/provider_source.go b/provider_source.go index bdda08bd3..b65dbf439 100644 --- a/provider_source.go +++ b/provider_source.go @@ -39,6 +39,7 @@ func explicitProviderSource(config *cliconfig.ProviderInstallation, services *di var diags tfdiags.Diagnostics var searchRules []getproviders.MultiSourceSelector + log.Printf("[DEBUG] Explicit provider installation configuration is set") for _, methodConfig := range config.Methods { source, moreDiags := providerSourceForCLIConfigLocation(methodConfig.Location, services) diags = diags.Append(moreDiags) @@ -53,14 +54,16 @@ func explicitProviderSource(config *cliconfig.ProviderInstallation, services *di "Invalid provider source inclusion patterns", fmt.Sprintf("CLI config specifies invalid provider inclusion patterns: %s.", err), )) + continue } - exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Include) + exclude, err := getproviders.ParseMultiSourceMatchingPatterns(methodConfig.Exclude) if err != nil { diags = diags.Append(tfdiags.Sourceless( tfdiags.Error, "Invalid provider source exclusion patterns", fmt.Sprintf("CLI config specifies invalid provider exclusion patterns: %s.", err), )) + continue } searchRules = append(searchRules, getproviders.MultiSourceSelector{ @@ -68,6 +71,8 @@ func explicitProviderSource(config *cliconfig.ProviderInstallation, services *di Include: include, Exclude: exclude, }) + + log.Printf("[TRACE] Selected provider installation method %#v with includes %s and excludes %s", methodConfig.Location, include, exclude) } return getproviders.MultiSource(searchRules), diags From c6cbbcb79a751fcee2334b22bb3ee18aecaefb91 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 18:43:07 -0700 Subject: [PATCH 12/14] website: Initial documentation for provider_installation in the CLI config This is an initial draft of documentation for this new feature of the CLI configuration. This is mainly intended as a placeholder for now, because there are other documentation updates pending for the new provider namespacing and installation scheme and we'll likely want to revise these docs to better complement the broader documentation once it's written. --- .../docs/commands/cli-config.html.markdown | 219 ++++++++++++++++-- website/docs/configuration/providers.html.md | 128 ++-------- 2 files changed, 229 insertions(+), 118 deletions(-) diff --git a/website/docs/commands/cli-config.html.markdown b/website/docs/commands/cli-config.html.markdown index fd4e9a552..cd0a47cdf 100644 --- a/website/docs/commands/cli-config.html.markdown +++ b/website/docs/commands/cli-config.html.markdown @@ -51,6 +51,14 @@ disable_checkpoint = true The following settings can be set in the CLI configuration file: +- `credentials` - configures credentials for use with Terraform Cloud or + Terraform Enterprise. See [Credentials](#credentials) below for more + information. + +- `credentials_helper` - configures an external helper program for the storage + and retrieval of credentials for Terraform Cloud or Terraform Enterprise. + See [Credentials Helpers](#credentials-helpers) below for more information. + - `disable_checkpoint` — when set to `true`, disables [upgrade and security bulletin checks](/docs/commands/index.html#upgrade-and-security-bulletin-checks) that require reaching out to HashiCorp-provided network services. @@ -60,16 +68,12 @@ The following settings can be set in the CLI configuration file: id used to de-duplicate warning messages. - `plugin_cache_dir` — enables - [plugin caching](/docs/configuration/providers.html#provider-plugin-cache) + [plugin caching](#provider-plugin-cache) and specifies, as a string, the location of the plugin cache directory. -- `credentials` - configures credentials for use with Terraform Cloud or - Terraform Enterprise. See [Credentials](#credentials) below for more - information. - -- `credentials_helper` - configures an external helper program for the storage - and retrieval of credentials for Terraform Cloud or Terraform Enterprise. - See [Credentials Helpers](#credentials-helpers) below for more information. +- `provider_installation` - customizes the installation methods used by + `terraform init` when installing provider plugins. See + [Provider Installation](#provider-installation) below for more information. ## Credentials @@ -144,14 +148,199 @@ To learn how to write and install your own credentials helpers to integrate with existing in-house credentials management systems, see [the guide to Credentials Helper internals](/docs/internals/credentials-helpers.html). -## Deprecated Settings +## Provider Installation -The following settings are supported for backward compatibility but are no -longer recommended for use: +The default way to install provider plugins is from a provider registry. The +origin registry for a provider is encoded in the provider's source address, +like `registry.terraform.io/hashicorp/aws`. For convenience in the common case, +Terraform allows omitting the hostname portion for providers on +`registry.terraform.io`, so we'd normally write `hashicorp/aws` instead in +this case. + +Downloading a plugin directly from its origin registry is not always +appropriate, though. For example, the system where you are running Terraform +may not be able to access an origin registry due to firewall restrictions +within your organization or your locality. + +To allow using Terraform in these situations, there are some alternative +options for making provider plugins available to Terraform. + +### Explicit Installation Method Configuration + +A `provider_installation` block in the CLI configuration allows overriding +Terraform's default installation behaviors, so you can force Terraform to use +a local mirror for some or all of the providers you intend to use. + +The general structure of a `provider_installation` block is as follows: + +```hcl +provider_installation { + filesystem_mirror { + path = "/usr/share/terraform/providers" + include = ["example.com/*/*"] + } + direct { + exclude = ["example.com/*/*"] + } +} +``` + +Each of the nested blocks inside the `provider_installation` block specifies +one installation method. Each installation method can take both `include` +and `exclude` patterns that specify which providers a particular installation +method can be used for. In the example above, we specify that any provider +whose origin registry is at `example.com` can be installed only from the +filesystem mirror at `/usr/share/terraform/providers`, while all other +providers can be installed only directly from their origin registries. + +If you set both both `include` and `exclude` for a particular installation +method, the exclusion patterns take priority. For example, including +`registry.terraform.io/hashicorp/*` but also excluding +`registry.terraform.io/hashicorp/dns` will make that installation method apply +to everything in the `hashicorp` namespace with the exception of +`hashicorp/dns`. + +As with provider source addresses in the main configuration, you can omit +the `registry.terraform.io/` prefix for providers distributed through the +public Terraform registry, even when using wildcards. For example, +`registry.terraform.io/hashicorp/*` and `hashicorp/*` are equivalent. +`*/*` is a shorthand for `registry.terraform.io/*/*`, not for +`*/*/*`. + +The following are the two supported installation method types: + +* `direct`: request information about the provider directly from its origin + registry and download over the network from the location that registry + indicates. This method expects no additional arguments. + +* `filesystem_mirror`: consult a directory on the local disk for copies of + providers. This method requires the additional argument `path` to indicate + which directory to look in. + + Terraform expects the given directory to contain a nested directory structure + where the path segments together provide metadata about the available + providers. The following two directory structures are supported: + + * Packed layout: `HOSTNAME/NAMESPACE/TYPE/terraform-provider-TYPE_VERSION_TARGET.zip` + is the distribution zip file obtained from the provider's origin registry. + * Unpacked layout: `HOSTNAME/NAMESPACE/TYPE/VERSION/TARGET` is a directory + containing the result of extracting the provider's distribution zip file. + + In both layouts, the `VERSION` is a string like `2.0.0` and the `TARGET` + specifies a particular target platform using a format like `darwin_amd64`, + `linux_arm`, `windows_amd64`, etc. + + If you use the unpacked layout, Terraform will attempt to create a symbolic + link to the mirror directory when installing the provider, rather than + creating a deep copy of the directory. The packed layout prevents this + because Terraform must extract the zip file during installation. + + You can include multiple `filesystem_mirror` blocks in order to specify + several different directories to search. + +Terraform will try all of the specified methods whose include and exclude +patterns match a given provider, and select the newest version available across +all of those methods that matches the version constraint given in each +Terraform configuration. If you have a local mirror of a particular provider +and intend Terraform to use that local mirror exclusively, you must either +remove the `direct` installation method altogether or use its `exclude` +argument to disable its use for specific providers. + +### Implied Local Mirror Directories + +If your CLI configuration does not include a `provider_installation` block at +all, Terraform produces an _implied_ configuration. The implied configuration +includes a selection of `filesystem_mirror` methods and then the `direct` +method. + +The set of directories Terraform can select as filesystem mirrors depends on +the operating system where you are running Terraform: + +* **Windows:** `%APPDATA%/HashiCorp/Terraform/plugins` +* **Mac OS X:** `~/Library/Application Support/io.terraform/plugins` and + `/Library/Application Support/io.terraform/plugins` +* **Linux and other Unix-like systems**: Terraform implements the + [XDG Base Directory](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) + specification and appends `terraform/plugins` to all of the specified + data directories. Without any XDG environment variables set, Terraform + will use `~/.local/share/terraform/plugins`, + `/usr/local/share/terraform/plugins`, and `/usr/share/terraform/plugins`. + +Terraform will create an implied `filesystem_mirror` method block for each of +the directories indicated above that exists when Terraform starts up. + +In addition to the zero or more implied `filesystem_mirror` blocks, Terraform +also creates an implied `direct` block. Terraform will scan all of the +filesystem mirror directories to see which providers are placed there and +automatically exclude all of those providers from the implied `direct` block. +(This automatic `exclude` behavior applies only to _implicit_ `direct` blocks; +if you use explicit `provider_installation` you will need to write the intended +exclusions out yourself.) + +### Provider Plugin Cache + +By default, `terraform init` downloads plugins into a subdirectory of the +working directory so that each working directory is self-contained. As a +consequence, if you have multiple configurations that use the same provider +then a separate copy of its plugin will be downloaded for each configuration. + +Given that provider plugins can be quite large (on the order of hundreds of +megabytes), this default behavior can be inconvenient for those with slow +or metered Internet connections. Therefore Terraform optionally allows the +use of a local directory as a shared plugin cache, which then allows each +distinct plugin binary to be downloaded only once. + +To enable the plugin cache, use the `plugin_cache_dir` setting in +the CLI configuration file. For example: + +```hcl +plugin_cache_dir = "$HOME/.terraform.d/plugin-cache" +``` + +This directory must already exist before Terraform will cache plugins; +Terraform will not create the directory itself. + +Please note that on Windows it is necessary to use forward slash separators +(`/`) rather than the conventional backslash (`\`) since the configuration +file parser considers a backslash to begin an escape sequence. + +Setting this in the configuration file is the recommended approach for a +persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment +variable can be used to enable caching or to override an existing cache +directory within a particular shell session: + +```bash +export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" +``` + +When a plugin cache directory is enabled, the `terraform init` command will +still use the configured or implied installation methods to obtain metadata +about which plugins are available, but once a suitable version has been +selected it will first check to see if the chosen plugin is already available +in the cache directory. If so, Terraform will use the previously-downloaded +copy. + +If the selected plugin is not already in the cache, Terraform will download +it into the cache first and then copy it from there into the correct location +under your current working directory. When possible Terraform will use +symbolic links to avoid storing a separate copy of a cached plugin in multiple +directories. + +The plugin cache directory _must not_ also be one of the configured or implied +filesystem mirror directories, since the cache management logic conflicts with +the filesystem mirror logic when operating on the same directory. + +Terraform will never itself delete a plugin from the plugin cache once it has +been placed there. Over time, as plugins are upgraded, the cache directory may +grow to contain several unused versions which you must delete manually. + +## Removed Settings + +The following settings are supported in Terraform 0.12 and earlier but are +no longer recommended for use: * `providers` - a configuration block that allows specifying the locations of specific plugins for each named provider. This mechanism is deprecated - because it is unable to specify a version number for each plugin, and thus - it does not co-operate with the plugin versioning mechanism. Instead, - place the plugin executable files in - [the third-party plugins directory](/docs/configuration/providers.html#third-party-plugins). + because it is unable to specify a version number and source for each provider. + See [Provider Installation](#provider-installation) above for the replacement + of this setting in Terraform 0.13 and later. diff --git a/website/docs/configuration/providers.html.md b/website/docs/configuration/providers.html.md index dff4f6cc9..4d06e2066 100644 --- a/website/docs/configuration/providers.html.md +++ b/website/docs/configuration/providers.html.md @@ -233,65 +233,35 @@ from their parents. Anyone can develop and distribute their own Terraform providers. (See [Writing Custom Providers](/docs/extend/writing-custom-providers.html) for more -about provider development.) These third-party providers must be manually -installed, since `terraform init` cannot automatically download them. +about provider development.) -Install third-party providers by placing their plugin executables in the user -plugins directory. The user plugins directory is in one of the following -locations, depending on the host operating system: +The main way to distribute a provider is via a provider registry, and the main +provider registry is +[part of the public Terraform Registry](https://registry.terraform.io/browse/providers), +along with public shared modules. -Operating system | User plugins directory -------------------|----------------------- -Windows | `%APPDATA%\terraform.d\plugins` -All other systems | `~/.terraform.d/plugins` +Providers distributed via a public registry to not require any special +additional configuration to use, once you know their source addresses. You can +specify both official and third-party source addresses in the +`required_providers` block in your module: -Once a plugin is installed, `terraform init` can initialize it normally. You must run this command from the directory where the configuration files are located. +```hcl +terraform { + required_providers { + # An example third-party provider. Not actually available. + example = { + source = "example.com/examplecorp/example" + } + } +} +``` -Providers distributed by HashiCorp can also go in the user plugins directory. If -a manually installed version meets the configuration's version constraints, -Terraform will use it instead of downloading that provider. This is useful in -airgapped environments and when testing pre-release provider builds. - -### Plugin Names and Versions - -The naming scheme for provider plugins is `terraform-provider-_vX.Y.Z`, -and Terraform uses the name to understand the name and version of a particular -provider binary. - -If multiple versions of a plugin are installed, Terraform will use the newest -version that meets the configuration's version constraints. - -Third-party plugins are often distributed with an appropriate filename already -set in the distribution archive, so that they can be extracted directly into the -user plugins directory. - -### OS and Architecture Directories - -Terraform plugins are compiled for a specific operating system and architecture, -and any plugins in the root of the user plugins directory must be compiled for -the current system. - -If you use the same plugins directory on multiple systems, you can install -plugins into subdirectories with a naming scheme of `_` (for example, -`darwin_amd64`). Terraform uses plugins from the root of the plugins directory -and from the subdirectory that corresponds to the current system, ignoring -other subdirectories. - -Terraform's OS and architecture strings are the standard ones used by the Go -language. The following are the most common: - -* `darwin_amd64` -* `freebsd_386` -* `freebsd_amd64` -* `freebsd_arm` -* `linux_386` -* `linux_amd64` -* `linux_arm` -* `openbsd_386` -* `openbsd_amd64` -* `solaris_amd64` -* `windows_386` -* `windows_amd64` +Installing directly from a registry is not appropriate for all situations, +though. If you are running Terraform from a system that cannot access some or +all of the necessary origin registries, you can configure Terraform to obtain +providers from a local mirror instead. For more information, see +[Provider Installation](../commands/cli-config.html#provider-installation) +in the CLI configuration documentation. ## Provider Plugin Cache @@ -308,51 +278,3 @@ distinct plugin binary to be downloaded only once. To enable the plugin cache, use the `plugin_cache_dir` setting in [the CLI configuration file](/docs/commands/cli-config.html). -For example: - -```hcl -# (Note that the CLI configuration file is _not_ the same as the .tf files -# used to configure infrastructure.) - -plugin_cache_dir = "$HOME/.terraform.d/plugin-cache" -``` - -This directory must already exist before Terraform will cache plugins; -Terraform will not create the directory itself. - -Please note that on Windows it is necessary to use forward slash separators -(`/`) rather than the conventional backslash (`\`) since the configuration -file parser considers a backslash to begin an escape sequence. - -Setting this in the configuration file is the recommended approach for a -persistent setting. Alternatively, the `TF_PLUGIN_CACHE_DIR` environment -variable can be used to enable caching or to override an existing cache -directory within a particular shell session: - -```bash -export TF_PLUGIN_CACHE_DIR="$HOME/.terraform.d/plugin-cache" -``` - -When a plugin cache directory is enabled, the `terraform init` command will -still access the plugin distribution server to obtain metadata about which -plugins are available, but once a suitable version has been selected it will -first check to see if the selected plugin is already available in the cache -directory. If so, the already-downloaded plugin binary will be used. - -If the selected plugin is not already in the cache, it will be downloaded -into the cache first and then copied from there into the correct location -under your current working directory. - -When possible, Terraform will use hardlinks or symlinks to avoid storing -a separate copy of a cached plugin in multiple directories. At present, this -is not supported on Windows and instead a copy is always created. - -The plugin cache directory must _not_ be the third-party plugin directory -or any other directory Terraform searches for pre-installed plugins, since -the cache management logic conflicts with the normal plugin discovery logic -when operating on the same directory. - -Please note that Terraform will never itself delete a plugin from the -plugin cache once it's been placed there. Over time, as plugins are upgraded, -the cache directory may grow to contain several unused versions which must be -manually deleted. From dadec6ee9ed8b4a8b884dec3e49fbb27e0c5334b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 23 Apr 2020 10:50:50 -0700 Subject: [PATCH 13/14] fixup docs --- website/docs/commands/cli-config.html.markdown | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/docs/commands/cli-config.html.markdown b/website/docs/commands/cli-config.html.markdown index cd0a47cdf..9e76cb5ad 100644 --- a/website/docs/commands/cli-config.html.markdown +++ b/website/docs/commands/cli-config.html.markdown @@ -162,8 +162,9 @@ appropriate, though. For example, the system where you are running Terraform may not be able to access an origin registry due to firewall restrictions within your organization or your locality. -To allow using Terraform in these situations, there are some alternative -options for making provider plugins available to Terraform. +To allow using Terraform providers in these situations, there are some +alternative options for making provider plugins available to Terraform which +we'll describe in the following sections. ### Explicit Installation Method Configuration From 622abf707da2dea75e1a5d0fccb5acd25ff810c7 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 22 Apr 2020 18:53:12 -0700 Subject: [PATCH 14/14] command/cliconfig: Remove redundant struct types These were being used in an earlier iteration of the provider installation configuration but it was all collapsed down into a single ProviderInstallationMethod type later, making these redundant. --- command/cliconfig/cliconfig.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/command/cliconfig/cliconfig.go b/command/cliconfig/cliconfig.go index c765132d8..2aecfb9d1 100644 --- a/command/cliconfig/cliconfig.go +++ b/command/cliconfig/cliconfig.go @@ -63,22 +63,6 @@ type ConfigCredentialsHelper struct { Args []string `hcl:"args"` } -// ConfigProviderInstallationFilesystemMirror represents a "filesystem_mirror" -// block inside ConfigProviderInstallation. -type ConfigProviderInstallationFilesystemMirror struct { - Path string `hcl:"path"` - Include []string `hcl:"include"` - Exclude []string `hcl:"exclude"` -} - -// ConfigProviderInstallationNetworkMirror represents a "network_mirror" block -// inside ConfigProviderInstallation. -type ConfigProviderInstallationNetworkMirror struct { - Hostname string `hcl:"hostname"` - Include []string `hcl:"include"` - Exclude []string `hcl:"exclude"` -} - // BuiltinConfig is the built-in defaults for the configuration. These // can be overridden by user configurations. var BuiltinConfig Config