Merge #16543: Bring the "terraform" provider back into core

This commit is contained in:
Martin Atkins 2017-11-03 12:08:31 -07:00 committed by GitHub
commit 3b180d7f8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 827 additions and 3 deletions

View File

@ -0,0 +1,121 @@
package terraform
import (
"fmt"
"log"
"time"
"github.com/hashicorp/terraform/backend"
backendinit "github.com/hashicorp/terraform/backend/init"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
)
func dataSourceRemoteState() *schema.Resource {
return &schema.Resource{
Read: dataSourceRemoteStateRead,
Schema: map[string]*schema.Schema{
"backend": {
Type: schema.TypeString,
Required: true,
ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
if vStr, ok := v.(string); ok && vStr == "_local" {
ws = append(ws, "Use of the %q backend is now officially "+
"supported as %q. Please update your configuration to ensure "+
"compatibility with future versions of Terraform.",
"_local", "local")
}
return
},
},
"config": {
Type: schema.TypeMap,
Optional: true,
},
"defaults": {
Type: schema.TypeMap,
Optional: true,
},
"environment": {
Type: schema.TypeString,
Optional: true,
Default: backend.DefaultStateName,
},
"__has_dynamic_attributes": {
Type: schema.TypeString,
Optional: true,
},
},
}
}
func dataSourceRemoteStateRead(d *schema.ResourceData, meta interface{}) error {
backend := d.Get("backend").(string)
// Get the configuration in a type we want.
rawConfig, err := config.NewRawConfig(d.Get("config").(map[string]interface{}))
if err != nil {
return fmt.Errorf("error initializing backend: %s", err)
}
// Don't break people using the old _local syntax - but note warning above
if backend == "_local" {
log.Println(`[INFO] Switching old (unsupported) backend "_local" to "local"`)
backend = "local"
}
// Create the client to access our remote state
log.Printf("[DEBUG] Initializing remote state backend: %s", backend)
f := backendinit.Backend(backend)
if f == nil {
return fmt.Errorf("Unknown backend type: %s", backend)
}
b := f()
// Configure the backend
if err := b.Configure(terraform.NewResourceConfig(rawConfig)); err != nil {
return fmt.Errorf("error initializing backend: %s", err)
}
// Get the state
env := d.Get("environment").(string)
state, err := b.State(env)
if err != nil {
return fmt.Errorf("error loading the remote state: %s", err)
}
if err := state.RefreshState(); err != nil {
return err
}
d.SetId(time.Now().UTC().String())
outputMap := make(map[string]interface{})
defaults := d.Get("defaults").(map[string]interface{})
for key, val := range defaults {
outputMap[key] = val
}
remoteState := state.State()
if remoteState.Empty() {
log.Println("[DEBUG] empty remote state")
} else {
for key, val := range remoteState.RootModule().Outputs {
outputMap[key] = val.Value
}
}
mappedOutputs := remoteStateFlatten(outputMap)
for key, val := range mappedOutputs {
d.UnsafeSetFieldRaw(key, val)
}
return nil
}

View File

@ -0,0 +1,169 @@
package terraform
import (
"fmt"
"testing"
backendinit "github.com/hashicorp/terraform/backend/init"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/terraform"
)
func TestState_basic(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccState_basic,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue(
"data.terraform_remote_state.foo", "foo", "bar"),
),
},
},
})
}
func TestState_backends(t *testing.T) {
backendinit.Set("_ds_test", backendinit.Backend("local"))
defer backendinit.Set("_ds_test", nil)
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccState_backend,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue(
"data.terraform_remote_state.foo", "foo", "bar"),
),
},
},
})
}
func TestState_complexOutputs(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccState_complexOutputs,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue("terraform_remote_state.foo", "backend", "local"),
testAccCheckStateValue("terraform_remote_state.foo", "config.path", "./test-fixtures/complex_outputs.tfstate"),
testAccCheckStateValue("terraform_remote_state.foo", "computed_set.#", "2"),
testAccCheckStateValue("terraform_remote_state.foo", `map.%`, "2"),
testAccCheckStateValue("terraform_remote_state.foo", `map.key`, "test"),
),
},
},
})
}
func TestEmptyState_defaults(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccEmptyState_defaults,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue(
"data.terraform_remote_state.foo", "foo", "bar"),
),
},
},
})
}
func TestState_defaults(t *testing.T) {
resource.UnitTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
{
Config: testAccEmptyState_defaults,
Check: resource.ComposeTestCheckFunc(
testAccCheckStateValue(
"data.terraform_remote_state.foo", "foo", "bar"),
),
},
},
})
}
func testAccCheckStateValue(id, name, value string) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[id]
if !ok {
return fmt.Errorf("Not found: %s", id)
}
if rs.Primary.ID == "" {
return fmt.Errorf("No ID is set")
}
v := rs.Primary.Attributes[name]
if v != value {
return fmt.Errorf(
"Value for %s is %s, not %s", name, v, value)
}
return nil
}
}
const testAccState_basic = `
data "terraform_remote_state" "foo" {
backend = "local"
config {
path = "./test-fixtures/basic.tfstate"
}
}`
const testAccState_backend = `
data "terraform_remote_state" "foo" {
backend = "_ds_test"
config {
path = "./test-fixtures/basic.tfstate"
}
}`
const testAccState_complexOutputs = `
resource "terraform_remote_state" "foo" {
backend = "local"
config {
path = "./test-fixtures/complex_outputs.tfstate"
}
}`
const testAccEmptyState_defaults = `
data "terraform_remote_state" "foo" {
backend = "local"
config {
path = "./test-fixtures/empty.tfstate"
}
defaults {
foo = "bar"
}
}`
const testAccState_defaults = `
data "terraform_remote_state" "foo" {
backend = "local"
config {
path = "./test-fixtures/basic.tfstate"
}
defaults {
foo = "not bar"
}
}`

View File

@ -0,0 +1,76 @@
package terraform
import (
"fmt"
"reflect"
)
// remoteStateFlatten takes a structure and turns into a flat map[string]string.
//
// Within the "thing" parameter, only primitive values are allowed. Structs are
// not supported. Therefore, it can only be slices, maps, primitives, and
// any combination of those together.
//
// The difference between this version and the version in package flatmap is that
// we add the count key for maps in this version, and return a normal
// map[string]string instead of a flatmap.Map
func remoteStateFlatten(thing map[string]interface{}) map[string]string {
result := make(map[string]string)
for k, raw := range thing {
flatten(result, k, reflect.ValueOf(raw))
}
return result
}
func flatten(result map[string]string, prefix string, v reflect.Value) {
if v.Kind() == reflect.Interface {
v = v.Elem()
}
switch v.Kind() {
case reflect.Bool:
if v.Bool() {
result[prefix] = "true"
} else {
result[prefix] = "false"
}
case reflect.Int:
result[prefix] = fmt.Sprintf("%d", v.Int())
case reflect.Map:
flattenMap(result, prefix, v)
case reflect.Slice:
flattenSlice(result, prefix, v)
case reflect.String:
result[prefix] = v.String()
default:
panic(fmt.Sprintf("Unknown: %s", v))
}
}
func flattenMap(result map[string]string, prefix string, v reflect.Value) {
mapKeys := v.MapKeys()
result[fmt.Sprintf("%s.%%", prefix)] = fmt.Sprintf("%d", len(mapKeys))
for _, k := range mapKeys {
if k.Kind() == reflect.Interface {
k = k.Elem()
}
if k.Kind() != reflect.String {
panic(fmt.Sprintf("%s: map key is not string: %s", prefix, k))
}
flatten(result, fmt.Sprintf("%s.%s", prefix, k.String()), v.MapIndex(k))
}
}
func flattenSlice(result map[string]string, prefix string, v reflect.Value) {
prefix = prefix + "."
result[prefix+"#"] = fmt.Sprintf("%d", v.Len())
for i := 0; i < v.Len(); i++ {
flatten(result, fmt.Sprintf("%s%d", prefix, i), v.Index(i))
}
}

View File

@ -0,0 +1,21 @@
package terraform
import (
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
)
// Provider returns a terraform.ResourceProvider.
func Provider() terraform.ResourceProvider {
return &schema.Provider{
ResourcesMap: map[string]*schema.Resource{
"terraform_remote_state": schema.DataSourceResourceShim(
"terraform_remote_state",
dataSourceRemoteState(),
),
},
DataSourcesMap: map[string]*schema.Resource{
"terraform_remote_state": dataSourceRemoteState(),
},
}
}

View File

@ -0,0 +1,31 @@
package terraform
import (
"testing"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
)
var testAccProviders map[string]terraform.ResourceProvider
var testAccProvider *schema.Provider
func init() {
testAccProvider = Provider().(*schema.Provider)
testAccProviders = map[string]terraform.ResourceProvider{
"terraform": testAccProvider,
}
}
func TestProvider(t *testing.T) {
if err := Provider().(*schema.Provider).InternalValidate(); err != nil {
t.Fatalf("err: %s", err)
}
}
func TestProvider_impl(t *testing.T) {
var _ terraform.ResourceProvider = Provider()
}
func testAccPreCheck(t *testing.T) {
}

View File

@ -0,0 +1,7 @@
{
"version": 1,
"modules": [{
"path": ["root"],
"outputs": { "foo": "bar" }
}]
}

View File

@ -0,0 +1,88 @@
{
"version": 3,
"terraform_version": "0.7.0",
"serial": 3,
"modules": [
{
"path": [
"root"
],
"outputs": {
"computed_map": {
"sensitive": false,
"type": "map",
"value": {
"key1": "value1"
}
},
"computed_set": {
"sensitive": false,
"type": "list",
"value": [
"setval1",
"setval2"
]
},
"map": {
"sensitive": false,
"type": "map",
"value": {
"key": "test",
"test": "test"
}
},
"set": {
"sensitive": false,
"type": "list",
"value": [
"test1",
"test2"
]
}
},
"resources": {
"test_resource.main": {
"type": "test_resource",
"primary": {
"id": "testId",
"attributes": {
"computed_list.#": "2",
"computed_list.0": "listval1",
"computed_list.1": "listval2",
"computed_map.%": "1",
"computed_map.key1": "value1",
"computed_read_only": "value_from_api",
"computed_read_only_force_new": "value_from_api",
"computed_set.#": "2",
"computed_set.2337322984": "setval1",
"computed_set.307881554": "setval2",
"id": "testId",
"list_of_map.#": "2",
"list_of_map.0.%": "2",
"list_of_map.0.key1": "value1",
"list_of_map.0.key2": "value2",
"list_of_map.1.%": "2",
"list_of_map.1.key3": "value3",
"list_of_map.1.key4": "value4",
"map.%": "2",
"map.key": "test",
"map.test": "test",
"map_that_look_like_set.%": "2",
"map_that_look_like_set.12352223": "hello",
"map_that_look_like_set.36234341": "world",
"optional_computed_map.%": "0",
"required": "Hello World",
"required_map.%": "3",
"required_map.key1": "value1",
"required_map.key2": "value2",
"required_map.key3": "value3",
"set.#": "2",
"set.2326977762": "test1",
"set.331058520": "test2"
}
}
}
}
}
]
}

View File

@ -50,6 +50,37 @@ func TestInitProviders(t *testing.T) {
}
func TestInitProvidersInternal(t *testing.T) {
t.Parallel()
// This test should _not_ reach out anywhere because the "terraform"
// provider is internal to the core terraform binary.
fixturePath := filepath.Join("test-fixtures", "terraform-provider")
tf := e2e.NewBinary(terraformBin, fixturePath)
defer tf.Close()
stdout, stderr, err := tf.Run("init")
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if stderr != "" {
t.Errorf("unexpected stderr output:\n%s", stderr)
}
if !strings.Contains(stdout, "Terraform has been successfully initialized!") {
t.Errorf("success message is missing from output:\n%s", stdout)
}
if strings.Contains(stdout, "Downloading plugin for provider") {
// Shouldn't have downloaded anything with this config, because the
// provider is built in.
t.Errorf("provider download message appeared in output:\n%s", stdout)
}
}
func TestInitProviders_pluginCache(t *testing.T) {
t.Parallel()

View File

@ -0,0 +1,10 @@
provider "terraform" {
}
data "terraform_remote_state" "test" {
backend = "local"
config = {
path = "nothing.tfstate"
}
}

View File

@ -310,6 +310,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
))
missing := c.missingPlugins(available, requirements)
internal := c.internalProviders()
var errs error
if c.getPlugins {
@ -319,6 +320,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
}
for provider, reqd := range missing {
if _, isInternal := internal[provider]; isInternal {
// Ignore internal providers; they are not eligible for
// installation.
continue
}
_, err := c.providerInstaller.Get(provider, reqd.Versions)
if err != nil {
@ -376,7 +383,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
// again. If anything changes, other commands that use providers will
// fail with an error instructing the user to re-run this command.
available = c.providerPluginSet() // re-discover to see newly-installed plugins
chosen := choosePlugins(available, requirements)
chosen := choosePlugins(available, internal, requirements)
digests := map[string][]byte{}
for name, meta := range chosen {
digest, err := meta.SHA256()

View File

@ -13,6 +13,7 @@ import (
"strings"
plugin "github.com/hashicorp/go-plugin"
terraformProvider "github.com/hashicorp/terraform/builtin/providers/terraform"
tfplugin "github.com/hashicorp/terraform/plugin"
"github.com/hashicorp/terraform/plugin/discovery"
"github.com/hashicorp/terraform/terraform"
@ -25,12 +26,25 @@ import (
// each that satisfies the given constraints.
type multiVersionProviderResolver struct {
Available discovery.PluginMetaSet
// Internal is a map that overrides the usual plugin selection process
// for internal plugins. These plugins do not support version constraints
// (will produce an error if one is set). This should be used only in
// exceptional circumstances since it forces the provider's release
// schedule to be tied to that of Terraform Core.
Internal map[string]terraform.ResourceProviderFactory
}
func choosePlugins(avail discovery.PluginMetaSet, reqd discovery.PluginRequirements) map[string]discovery.PluginMeta {
func choosePlugins(avail discovery.PluginMetaSet, internal map[string]terraform.ResourceProviderFactory, reqd discovery.PluginRequirements) map[string]discovery.PluginMeta {
candidates := avail.ConstrainVersions(reqd)
ret := map[string]discovery.PluginMeta{}
for name, metas := range candidates {
// If the provider is in our internal map then we ignore any
// discovered plugins for it since these are dealt with separately.
if _, isInternal := internal[name]; isInternal {
continue
}
if len(metas) == 0 {
continue
}
@ -45,8 +59,17 @@ func (r *multiVersionProviderResolver) ResolveProviders(
factories := make(map[string]terraform.ResourceProviderFactory, len(reqd))
var errs []error
chosen := choosePlugins(r.Available, reqd)
chosen := choosePlugins(r.Available, r.Internal, reqd)
for name, req := range reqd {
if factory, isInternal := r.Internal[name]; isInternal {
if !req.Versions.Unconstrained() {
errs = append(errs, fmt.Errorf("provider.%s: this provider is built in to Terraform and so it does not support version constraints", name))
continue
}
factories[name] = factory
continue
}
if newest, available := chosen[name]; available {
digest, err := newest.SHA256()
if err != nil {
@ -233,6 +256,15 @@ func (m *Meta) providerPluginManuallyInstalledSet() discovery.PluginMetaSet {
func (m *Meta) providerResolver() terraform.ResourceProviderResolver {
return &multiVersionProviderResolver{
Available: m.providerPluginSet(),
Internal: m.internalProviders(),
}
}
func (m *Meta) internalProviders() map[string]terraform.ResourceProviderFactory {
return map[string]terraform.ResourceProviderFactory{
"terraform": func() (terraform.ResourceProvider, error) {
return terraformProvider.Provider(), nil
},
}
}

View File

@ -9,8 +9,91 @@ import (
"testing"
"github.com/hashicorp/terraform/plugin/discovery"
"github.com/hashicorp/terraform/terraform"
)
func TestMultiVersionProviderResolver(t *testing.T) {
available := make(discovery.PluginMetaSet)
available.Add(discovery.PluginMeta{
Name: "plugin",
Version: "1.0.0",
Path: "test-fixtures/empty-file",
})
resolver := &multiVersionProviderResolver{
Internal: map[string]terraform.ResourceProviderFactory{
"internal": func() (terraform.ResourceProvider, error) {
return &terraform.MockResourceProvider{
ResourcesReturn: []terraform.ResourceType{
{
Name: "internal_foo",
},
},
}, nil
},
},
Available: available,
}
t.Run("plugin matches", func(t *testing.T) {
reqd := discovery.PluginRequirements{
"plugin": &discovery.PluginConstraints{
Versions: discovery.ConstraintStr("1.0.0").MustParse(),
},
}
got, err := resolver.ResolveProviders(reqd)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if ct := len(got); ct != 1 {
t.Errorf("wrong number of results %d; want 1", ct)
}
if _, exists := got["plugin"]; !exists {
t.Errorf("provider \"plugin\" not in result")
}
})
t.Run("plugin doesn't match", func(t *testing.T) {
reqd := discovery.PluginRequirements{
"plugin": &discovery.PluginConstraints{
Versions: discovery.ConstraintStr("2.0.0").MustParse(),
},
}
_, err := resolver.ResolveProviders(reqd)
if err == nil {
t.Errorf("resolved successfully, but want error")
}
})
t.Run("internal matches", func(t *testing.T) {
reqd := discovery.PluginRequirements{
"internal": &discovery.PluginConstraints{
Versions: discovery.AllVersions,
},
}
got, err := resolver.ResolveProviders(reqd)
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if ct := len(got); ct != 1 {
t.Errorf("wrong number of results %d; want 1", ct)
}
if _, exists := got["internal"]; !exists {
t.Errorf("provider \"internal\" not in result")
}
})
t.Run("internal with version constraint", func(t *testing.T) {
// Version constraints are not permitted for internal providers
reqd := discovery.PluginRequirements{
"internal": &discovery.PluginConstraints{
Versions: discovery.ConstraintStr("2.0.0").MustParse(),
},
}
_, err := resolver.ResolveProviders(reqd)
if err == nil {
t.Errorf("resolved successfully, but want error")
}
})
}
func TestPluginPath(t *testing.T) {
td, err := ioutil.TempDir("", "tf")
if err != nil {
@ -36,6 +119,26 @@ func TestPluginPath(t *testing.T) {
}
}
func TestInternalProviders(t *testing.T) {
m := Meta{}
internal := m.internalProviders()
tfProvider, err := internal["terraform"]()
if err != nil {
t.Fatal(err)
}
dataSources := tfProvider.DataSources()
found := false
for _, ds := range dataSources {
if ds.Name == "terraform_remote_state" {
found = true
}
}
if !found {
t.Errorf("didn't find terraform_remote_state in internal \"terraform\" provider")
}
}
// mockProviderInstaller is a discovery.PluginInstaller implementation that
// is a mock for discovery.ProviderInstaller.
type mockProviderInstaller struct {

View File

View File

@ -0,0 +1,70 @@
---
layout: "terraform"
page_title: "Terraform: terraform_remote_state"
sidebar_current: "docs-terraform-datasource-remote-state"
description: |-
Accesses state meta data from a remote backend.
---
# remote_state
Retrieves state meta data from a remote backend
## Example Usage
```hcl
data "terraform_remote_state" "vpc" {
backend = "atlas"
config {
name = "hashicorp/vpc-prod"
}
}
resource "aws_instance" "foo" {
# ...
subnet_id = "${data.terraform_remote_state.vpc.subnet_id}"
}
```
## Argument Reference
The following arguments are supported:
* `backend` - (Required) The remote backend to use.
* `environment` - (Optional) The Terraform environment to use.
* `config` - (Optional) The configuration of the remote backend.
* `defaults` - (Optional) default value for outputs in case state file is empty or it does not have the output.
* Remote state config docs can be found [here](/docs/backends/types/terraform-enterprise.html)
## Attributes Reference
The following attributes are exported:
* `backend` - See Argument Reference above.
* `config` - See Argument Reference above.
In addition, each output in the remote state appears as a top level attribute
on the `terraform_remote_state` resource.
## Root Outputs Only
Only the root level outputs from the remote state are accessible. Outputs from
modules within the state cannot be accessed. If you want a module output to be
accessible via a remote state, you must thread the output through to a root
output.
An example is shown below:
```hcl
module "app" {
source = "..."
}
output "app_value" {
value = "${module.app.value}"
}
```
In this example, the output `value` from the "app" module is available as
"app_value". If this root level output hadn't been created, then a remote state
resource wouldn't be able to access the `value` output on the module.

View File

@ -0,0 +1,32 @@
---
layout: "terraform"
page_title: "Provider: Terraform"
sidebar_current: "docs-terraform-index"
description: |-
The Terraform provider is used to access meta data from shared infrastructure.
---
# Terraform Provider
The terraform provider provides access to outputs from the Terraform state
of shared infrastructure.
Use the navigation to the left to read about the available data sources.
## Example Usage
```hcl
# Shared infrastructure state stored in Atlas
data "terraform_remote_state" "vpc" {
backend = "atlas"
config {
name = "hashicorp/vpc-prod"
}
}
resource "aws_instance" "foo" {
# ...
subnet_id = "${data.terraform_remote_state.vpc.subnet_id}"
}
```

View File

@ -0,0 +1,26 @@
<% wrap_layout :inner do %>
<% content_for :sidebar do %>
<div class="docs-sidebar hidden-print affix-top" role="complementary">
<ul class="nav docs-sidenav">
<li<%= sidebar_current("docs-home") %>>
<a href="/docs/providers/index.html">All Providers</a>
</li>
<li<%= sidebar_current("docs-terraform-index") %>>
<a href="/docs/providers/terraform/index.html">Terraform Provider</a>
</li>
<li<%= sidebar_current("docs-terraform-datasource") %>>
<a href="#">Data Sources</a>
<ul class="nav nav-visible">
<li<%= sidebar_current("docs-terraform-datasource-remote-state") %>>
<a href="/docs/providers/terraform/d/remote_state.html">terraform_remote_state</a>
</li>
</ul>
</li>
</ul>
</div>
<% end %>
<%= yield %>
<% end %>