"external" data source, for integrating with external programs (#8768)

* "external" provider for gluing in external logic

This provider will become a bit of glue to help people interface external
programs with Terraform without writing a full Terraform provider.

It will be nowhere near as capable as a first-class provider, but is
intended as a light-touch way to integrate some pre-existing or custom
system into Terraform.

* Unit test for the "resourceProvider" utility function

This small function determines the dependable name of a provider for
a given resource name and optional provider alias. It's simple but it's
a key part of how resource nodes get connected to provider nodes so
worth specifying the intended behavior in the form of a test.

* Allow a provider to export a resource with the provider's name

If a provider only implements one resource of each type (managed vs. data)
then it can be reasonable for the resource names to exactly match the
provider name, if the provider name is descriptive enough for the
purpose of the each resource to be obvious.

* provider/external: data source

A data source that executes a child process, expecting it to support a
particular gateway protocol, and exports its result. This can be used as
a straightforward way to retrieve data from sources that Terraform
doesn't natively support..

* website: documentation for the "external" provider

builtin/bins/provider-external/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"github.com/hashicorp/terraform/builtin/providers/external"
+	"github.com/hashicorp/terraform/plugin"
+)
+
+func main() {
+	plugin.Serve(&plugin.ServeOpts{
+		ProviderFunc: external.Provider,
+	})
+}

builtin/providers/external/data_source.go

@@ -0,0 +1,93 @@
+package external
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+
+	"github.com/hashicorp/terraform/helper/schema"
+)
+
+func dataSource() *schema.Resource {
+	return &schema.Resource{
+		Read: dataSourceRead,
+
+		Schema: map[string]*schema.Schema{
+			"program": &schema.Schema{
+				Type:     schema.TypeList,
+				Required: true,
+				Elem: &schema.Schema{
+					Type: schema.TypeString,
+				},
+			},
+
+			"query": &schema.Schema{
+				Type:     schema.TypeMap,
+				Optional: true,
+				Elem: &schema.Schema{
+					Type: schema.TypeString,
+				},
+			},
+
+			"result": &schema.Schema{
+				Type:     schema.TypeMap,
+				Computed: true,
+				Elem: &schema.Schema{
+					Type: schema.TypeString,
+				},
+			},
+		},
+	}
+}
+
+func dataSourceRead(d *schema.ResourceData, meta interface{}) error {
+
+	programI := d.Get("program").([]interface{})
+	query := d.Get("query").(map[string]interface{})
+
+	// This would be a ValidateFunc if helper/schema allowed these
+	// to be applied to lists.
+	if err := validateProgramAttr(programI); err != nil {
+		return err
+	}
+
+	program := make([]string, len(programI))
+	for i, vI := range programI {
+		program[i] = vI.(string)
+	}
+
+	cmd := exec.Command(program[0], program[1:]...)
+
+	queryJson, err := json.Marshal(query)
+	if err != nil {
+		// Should never happen, since we know query will always be a map
+		// from string to string, as guaranteed by d.Get and our schema.
+		return err
+	}
+
+	cmd.Stdin = bytes.NewReader(queryJson)
+
+	resultJson, err := cmd.Output()
+	if err != nil {
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			if exitErr.Stderr != nil && len(exitErr.Stderr) > 0 {
+				return fmt.Errorf("failed to execute %q: %s", program[0], string(exitErr.Stderr))
+			}
+			return fmt.Errorf("command %q failed with no error message", program[0])
+		} else {
+			return fmt.Errorf("failed to execute %q: %s", program[0], err)
+		}
+	}
+
+	result := map[string]string{}
+	err = json.Unmarshal(resultJson, &result)
+	if err != nil {
+		return fmt.Errorf("command %q produced invalid JSON: %s", program[0], err)
+	}
+
+	d.Set("result", result)
+
+	d.SetId("-")
+	return nil
+}

builtin/providers/external/data_source_test.go

@@ -0,0 +1,124 @@
+package external
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/terraform/helper/resource"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+const testDataSourceConfig_basic = `
+data "external" "test" {
+  program = ["%s", "cheese"]
+
+  query = {
+    value = "pizza"
+  }
+}
+
+output "query_value" {
+  value = "${data.external.test.result["query_value"]}"
+}
+
+output "argument" {
+  value = "${data.external.test.result["argument"]}"
+}
+`
+
+func TestDataSource_basic(t *testing.T) {
+	programPath, err := buildDataSourceTestProgram()
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+
+	resource.UnitTest(t, resource.TestCase{
+		Providers: testProviders,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: fmt.Sprintf(testDataSourceConfig_basic, programPath),
+				Check: func(s *terraform.State) error {
+					_, ok := s.RootModule().Resources["data.external.test"]
+					if !ok {
+						return fmt.Errorf("missing data resource")
+					}
+
+					outputs := s.RootModule().Outputs
+
+					if outputs["argument"] == nil {
+						return fmt.Errorf("missing 'argument' output")
+					}
+					if outputs["query_value"] == nil {
+						return fmt.Errorf("missing 'query_value' output")
+					}
+
+					if outputs["argument"].Value != "cheese" {
+						return fmt.Errorf(
+							"'argument' output is %q; want 'cheese'",
+							outputs["argument"].Value,
+						)
+					}
+					if outputs["query_value"].Value != "pizza" {
+						return fmt.Errorf(
+							"'query_value' output is %q; want 'pizza'",
+							outputs["query_value"].Value,
+						)
+					}
+
+					return nil
+				},
+			},
+		},
+	})
+}
+
+const testDataSourceConfig_error = `
+data "external" "test" {
+  program = ["%s"]
+
+  query = {
+    fail = "true"
+  }
+}
+`
+
+func TestDataSource_error(t *testing.T) {
+	programPath, err := buildDataSourceTestProgram()
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+
+	resource.UnitTest(t, resource.TestCase{
+		Providers: testProviders,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config:      fmt.Sprintf(testDataSourceConfig_error, programPath),
+				ExpectError: regexp.MustCompile("I was asked to fail"),
+			},
+		},
+	})
+}
+
+func buildDataSourceTestProgram() (string, error) {
+	// We have a simple Go program that we use as a stub for testing.
+	cmd := exec.Command(
+		"go", "install",
+		"github.com/hashicorp/terraform/builtin/providers/external/test-programs/tf-acc-external-data-source",
+	)
+	err := cmd.Run()
+
+	if err != nil {
+		return "", fmt.Errorf("failed to build test stub program: %s", err)
+	}
+
+	programPath := path.Join(
+		os.Getenv("GOPATH"), "bin", "tf-acc-external-data-source",
+	)
+	return programPath, nil
+}

builtin/providers/external/provider.go

@@ -0,0 +1,15 @@
+package external
+
+import (
+	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func Provider() terraform.ResourceProvider {
+	return &schema.Provider{
+		DataSourcesMap: map[string]*schema.Resource{
+			"external": dataSource(),
+		},
+		ResourcesMap: map[string]*schema.Resource{},
+	}
+}

builtin/providers/external/provider_test.go

@@ -0,0 +1,18 @@
+package external
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func TestProvider(t *testing.T) {
+	if err := Provider().(*schema.Provider).InternalValidate(); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+var testProviders = map[string]terraform.ResourceProvider{
+	"external": Provider(),
+}

builtin/providers/external/test-programs/tf-acc-external-data-source/main.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+// This is a minimal implementation of the external data source protocol
+// intended only for use in the provider acceptance tests.
+//
+// In practice it's likely not much harder to just write a real Terraform
+// plugin if you're going to be writing your data source in Go anyway;
+// this example is just in Go because we want to avoid introducing
+// additional language runtimes into the test environment.
+func main() {
+	queryBytes, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		panic(err)
+	}
+
+	var query map[string]string
+	err = json.Unmarshal(queryBytes, &query)
+	if err != nil {
+		panic(err)
+	}
+
+	if query["fail"] != "" {
+		fmt.Fprintf(os.Stderr, "I was asked to fail\n")
+		os.Exit(1)
+	}
+
+	var result = map[string]string{
+		"result":      "yes",
+		"query_value": query["value"],
+	}
+
+	if len(os.Args) >= 2 {
+		result["argument"] = os.Args[1]
+	}
+
+	resultBytes, err := json.Marshal(result)
+	if err != nil {
+		panic(err)
+	}
+
+	os.Stdout.Write(resultBytes)
+	os.Exit(0)
+}

builtin/providers/external/util.go

@@ -0,0 +1,35 @@
+package external
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+// validateProgramAttr is a validation function for the "program" attribute we
+// accept as input on our resources.
+//
+// The attribute is assumed to be specified in schema as a list of strings.
+func validateProgramAttr(v interface{}) error {
+	args := v.([]interface{})
+	if len(args) < 1 {
+		return fmt.Errorf("'program' list must contain at least one element")
+	}
+
+	for i, vI := range args {
+		if _, ok := vI.(string); !ok {
+			return fmt.Errorf(
+				"'program' element %d is %T; a string is required",
+				i, vI,
+			)
+		}
+	}
+
+	// first element is assumed to be an executable command, possibly found
+	// using the PATH environment variable.
+	_, err := exec.LookPath(args[0].(string))
+	if err != nil {
+		return fmt.Errorf("can't find external program %q", args[0])
+	}
+
+	return nil
+}

command/internal_plugin_list.go

@@ -24,6 +24,7 @@ import (
 	dnsimpleprovider "github.com/hashicorp/terraform/builtin/providers/dnsimple"
 	dockerprovider "github.com/hashicorp/terraform/builtin/providers/docker"
 	dynprovider "github.com/hashicorp/terraform/builtin/providers/dyn"
+	externalprovider "github.com/hashicorp/terraform/builtin/providers/external"
 	fastlyprovider "github.com/hashicorp/terraform/builtin/providers/fastly"
 	githubprovider "github.com/hashicorp/terraform/builtin/providers/github"
 	googleprovider "github.com/hashicorp/terraform/builtin/providers/google"
@@ -85,6 +86,7 @@ var InternalProviders = map[string]plugin.ProviderFunc{
 	"dnsimple":     dnsimpleprovider.Provider,
 	"docker":       dockerprovider.Provider,
 	"dyn":          dynprovider.Provider,
+	"external":     externalprovider.Provider,
 	"fastly":       fastlyprovider.Provider,
 	"github":       githubprovider.Provider,
 	"google":       googleprovider.Provider,

terraform/util.go

@@ -54,7 +54,10 @@ func resourceProvider(t, alias string) string {
 
 	idx := strings.IndexRune(t, '_')
 	if idx == -1 {
-		return ""
+		// If no underscores, the resource name is assumed to be
+		// also the provider name, e.g. if the provider exposes
+		// only a single resource of each type.
+		return t
 	}
 
 	return t[:idx]

terraform/util_test.go

@@ -46,3 +46,53 @@ func TestStrSliceContains(t *testing.T) {
 		t.Fatalf("Bad")
 	}
 }
+
+func TestUtilResourceProvider(t *testing.T) {
+	type testCase struct {
+		ResourceName string
+		Alias        string
+		Expected     string
+	}
+
+	tests := []testCase{
+		{
+			// If no alias is provided, the first underscore-separated segment
+			// is assumed to be the provider name.
+			ResourceName: "aws_thing",
+			Alias:        "",
+			Expected:     "aws",
+		},
+		{
+			// If we have more than one underscore then it's the first one that we'll use.
+			ResourceName: "aws_thingy_thing",
+			Alias:        "",
+			Expected:     "aws",
+		},
+		{
+			// A provider can export a resource whose name is just the bare provider name,
+			// e.g. because the provider only has one resource and so any additional
+			// parts would be redundant.
+			ResourceName: "external",
+			Alias:        "",
+			Expected:     "external",
+		},
+		{
+			// Alias always overrides the default extraction of the name
+			ResourceName: "aws_thing",
+			Alias:        "tls.baz",
+			Expected:     "tls.baz",
+		},
+	}
+
+	for _, test := range tests {
+		got := resourceProvider(test.ResourceName, test.Alias)
+		if got != test.Expected {
+			t.Errorf(
+				"(%q, %q) produced %q; want %q",
+				test.ResourceName, test.Alias,
+				got,
+				test.Expected,
+			)
+		}
+	}
+}

website/source/assets/stylesheets/_docs.scss

@@ -25,6 +25,7 @@ body.layout-dme,
 body.layout-dnsimple,
 body.layout-docker,
 body.layout-dyn,
+body.layout-external,
 body.layout-github,
 body.layout-grafana,
 body.layout-fastly,

website/source/docs/providers/external/data_source.html.md

@@ -0,0 +1,116 @@
+---
+layout: "external"
+page_title: "External Data Source"
+sidebar_current: "docs-external-data-source"
+description: |-
+  Executes an external program that implements a data source.
+---
+
+# External Data Source
+
+The `external` data source allows an external program implementing a specific
+protocol (defined below) to act as a data source, exposing arbitrary
+data for use elsewhere in the Terraform configuration.
+
+~> **Warning** This mechanism is provided as an "escape hatch" for exceptional
+situations where a first-class Terraform provider is not more appropriate.
+Its capabilities are limited in comparison to a true data source, and
+implementing a data source via an external program is likely to hurt the
+portability of your Terraform configuration by creating dependencies on
+external programs and libraries that may not be available (or may need to
+be used differently) on different operating systems.
+
+~> **Warning** Terraform Enterprise does not guarantee availability of any
+particular language runtimes or external programs beyond standard shell
+utilities, so it is not recommended to use this data source within
+configurations that are applied within Terraform Enterprise.
+
+## Example Usage
+
+```
+data "external" "example" {
+  program = ["python", "${path.module}/example-data-source.py"]
+
+  query = {
+    # arbitrary map from strings to strings, passed
+    # to the external program as the data query.
+    id = "abc123"
+  }
+}
+```
+
+## External Program Protocol
+
+The external program described by the `program` attribute must implement a
+specific protocol for interacting with Terraform, as follows.
+
+The program must read all of the data passed to it on `stdin`, and parse
+it as a JSON object. The JSON object contains the contents of the `query`
+argument and its values will always be strings.
+
+The program must then produce a valid JSON object on `stdout`, which will
+be used to populate the `result` attribute exported to the rest of the
+Terraform configuration. This JSON object must again have all of its
+values as strings. On successful completion it must exit with status zero.
+
+If the program encounters an error and is unable to produce a result, it
+must print a human-readable error message (ideally a single line) to `stderr`
+and exit with a non-zero status. Any data on `stdout` is ignored if the
+program returns a non-zero status.
+
+All environment variables visible to the Terraform process are passed through
+to the child program.
+
+Terraform expects a data source to have *no observable side-effects*, and will
+re-run the program each time the state is refreshed.
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `program` - (Required) A list of strings, whose first element is the program
+  to run and whose subsequent elements are optional command line arguments
+  to the program. Terraform does not execute the program through a shell, so
+  it is not necessary to escape shell metacharacters nor add quotes around
+  arguments containing spaces.
+
+* `query` - (Optional) A map of string values to pass to the external program
+  as the query arguments. If not supplied, the program will recieve an empty
+  object as its input.
+
+## Attributes Reference
+
+The following attributes are exported:
+
+* `result` - A map of string values returned from the external program.
+
+## Processing JSON in shell scripts
+
+Since the external data source protocol uses JSON, it is recommended to use
+the utility [`jq`](https://stedolan.github.io/jq/) to translate to and from
+JSON in a robust way when implementing a data source in a shell scripting
+language.
+
+The following example shows some input/output boilerplate code for a
+data source implemented in bash:
+
+```
+#!/bin/bash
+
+# Exit if any of the intermediate steps fail
+set -e
+
+# Extract "foo" and "baz" arguments from the input into
+# FOO and BAZ shell variables.
+# jq will ensure that the values are properly quoted
+# and escaped for consumption by the shell.
+eval "$(jq -r '@sh "FOO=\(.foo) BAZ=\(.baz)"')"
+
+# Placeholder for whatever data-fetching logic your script implements
+FOOBAZ="$FOO BAZ"
+
+# Safely produce a JSON object containing the result value.
+# jq will ensure that the value is properly quoted
+# and escaped to produce a valid JSON string.
+jq -n --arg foobaz "$FOOBAZ" '{"foobaz":$foobaz}'
+```

website/source/docs/providers/external/index.html.markdown

@@ -0,0 +1,27 @@
+---
+layout: "external"
+page_title: "Provider: External"
+sidebar_current: "docs-external-index"
+description: |-
+  The external provider allows external scripts to be integrated with Terraform.
+---
+
+# External Provider
+
+`external` is a special provider that exists to provide an interface
+between Terraform and external programs.
+
+Using this provider it is possible to write separate programs that can
+participate in the Terraform workflow by implementing a specific protocol.
+
+This provider is intended to be used for simple situations where you wish
+to integrate Terraform with a system for which a first-class provider
+doesn't exist. It is not as powerful as a first-class Terraform provider,
+so users of this interface should carefully consider the implications
+described on each of the child documentation pages (available from the
+navigation bar) for each type of object this provider supports.
+
+~> **Warning** Terraform Enterprise does not guarantee availability of any
+particular language runtimes or external programs beyond standard shell
+utilities, so it is not recommended to use this provider within configurations
+that are applied within Terraform Enterprise.

website/source/layouts/docs.erb

@@ -238,6 +238,10 @@
 					<a href="/docs/providers/dyn/index.html">Dyn</a>
 					</li>
 
+					<li<%= sidebar_current("docs-providers-external") %>>
+					<a href="/docs/providers/external/index.html">External</a>
+					</li>
+
 					<li<%= sidebar_current("docs-providers-github") %>>
 					<a href="/docs/providers/github/index.html">GitHub</a>
 					</li>

website/source/layouts/external.erb

@@ -0,0 +1,22 @@
+<% wrap_layout :inner do %>
+	<% content_for :sidebar do %>
+		<div class="docs-sidebar hidden-print affix-top" role="complementary">
+			<ul class="nav docs-sidenav">
+				<li<%#= sidebar_current("docs-home") %>>
+					<a href="/docs/providers/index.html">&laquo; Documentation Home</a>
+				</li>
+
+				<li<%= sidebar_current("docs-external-index") %>>
+					<a href="/docs/providers/external/index.html">External Provider</a>
+					<ul class="nav nav-visible">
+						<li<%= sidebar_current("docs-external-data-source") %>>
+							<a href="/docs/providers/external/data_source.html">Data Source</a>
+						</li>
+					</ul>
+				</li>
+			</ul>
+		</div>
+	<% end %>
+
+	<%= yield %>
+<% end %>