core: New ResourceProvider methods for data resources

This is a breaking change to the ResourceProvider interface that adds the
new operations relating to data sources.

DataSources, ValidateDataSource, ReadDataDiff and ReadDataApply are the
data source equivalents of Resources, Validate, Diff and Apply (respectively)
for managed resources.

The diff/apply model seems at first glance a rather strange workflow for
read-only resources, but implementing data resources in this way allows them
to fit cleanly into the standard plan/apply lifecycle in cases where the
configuration contains computed arguments and thus the read must be deferred
until apply time.

Along with breaking the interface, we also fix up the plugin client/server
and helper/schema implementations of it, which are all of the callers
used when provider plugins use helper/schema. This would be a breaking
change for any provider plugin that directly implements the provider
interface, but no known plugins do this and it is not recommended.

At the helper/schema layer the implementer sees ReadDataApply as a "Read",
as opposed to "Create" or "Update" as in the managed resource Apply
implementation. The planning mechanics are handled entirely within
helper/schema, so that complexity is hidden from the provider implementation
itself.

helper/schema/provider.go

@@ -33,6 +33,14 @@ type Provider struct {
 	// Diff, etc. to the proper resource.
 	ResourcesMap map[string]*Resource
 
+	// DataSourcesMap is the collection of available data sources that
+	// this provider implements, with a Resource instance defining
+	// the schema and Read operation of each.
+	//
+	// Resource instances for data sources must have a Read function
+	// and must *not* implement Create, Update or Delete.
+	DataSourcesMap map[string]*Resource
+
 	// ConfigureFunc is a function for configuring the provider. If the
 	// provider doesn't need to be configured, this can be omitted.
 	//
@@ -68,7 +76,19 @@ func (p *Provider) InternalValidate() error {
 
 	for k, r := range p.ResourcesMap {
 		if err := r.InternalValidate(nil); err != nil {
-			return fmt.Errorf("%s: %s", k, err)
+			return fmt.Errorf("resource %s: %s", k, err)
+		}
+	}
+
+	for k, r := range p.DataSourcesMap {
+		if err := r.InternalValidate(nil); err != nil {
+			return fmt.Errorf("data source %s: %s", k, err)
+		}
+
+		if r.Create != nil || r.Update != nil || r.Delete != nil {
+			return fmt.Errorf(
+				"data source %s: must not have Create, Update or Delete", k,
+			)
 		}
 	}
 
@@ -262,3 +282,59 @@ func (p *Provider) ImportState(
 
 	return states, nil
 }
+
+// ValidateDataSource implementation of terraform.ResourceProvider interface.
+func (p *Provider) ValidateDataSource(
+	t string, c *terraform.ResourceConfig) ([]string, []error) {
+	r, ok := p.DataSourcesMap[t]
+	if !ok {
+		return nil, []error{fmt.Errorf(
+			"Provider doesn't support data source: %s", t)}
+	}
+
+	return r.Validate(c)
+}
+
+// ReadDataDiff implementation of terraform.ResourceProvider interface.
+func (p *Provider) ReadDataDiff(
+	info *terraform.InstanceInfo,
+	c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) {
+
+	r, ok := p.DataSourcesMap[info.Type]
+	if !ok {
+		return nil, fmt.Errorf("unknown data source: %s", info.Type)
+	}
+
+	return r.Diff(nil, c)
+}
+
+// RefreshData implementation of terraform.ResourceProvider interface.
+func (p *Provider) ReadDataApply(
+	info *terraform.InstanceInfo,
+	d *terraform.InstanceDiff) (*terraform.InstanceState, error) {
+
+	r, ok := p.DataSourcesMap[info.Type]
+	if !ok {
+		return nil, fmt.Errorf("unknown data source: %s", info.Type)
+	}
+
+	return r.ReadDataApply(d, p.meta)
+}
+
+// DataSources implementation of terraform.ResourceProvider interface.
+func (p *Provider) DataSources() []terraform.DataSource {
+	keys := make([]string, 0, len(p.DataSourcesMap))
+	for k, _ := range p.DataSourcesMap {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	result := make([]terraform.DataSource, 0, len(keys))
+	for _, k := range keys {
+		result = append(result, terraform.DataSource{
+			Name: k,
+		})
+	}
+
+	return result
+}

helper/schema/provider_test.go

@@ -132,6 +132,38 @@ func TestProviderResources(t *testing.T) {
 	}
 }
 
+func TestProviderDataSources(t *testing.T) {
+	cases := []struct {
+		P      *Provider
+		Result []terraform.DataSource
+	}{
+		{
+			P:      &Provider{},
+			Result: []terraform.DataSource{},
+		},
+
+		{
+			P: &Provider{
+				DataSourcesMap: map[string]*Resource{
+					"foo": nil,
+					"bar": nil,
+				},
+			},
+			Result: []terraform.DataSource{
+				terraform.DataSource{Name: "bar"},
+				terraform.DataSource{Name: "foo"},
+			},
+		},
+	}
+
+	for i, tc := range cases {
+		actual := tc.P.DataSources()
+		if !reflect.DeepEqual(actual, tc.Result) {
+			t.Fatalf("%d: got %#v; want %#v", i, actual, tc.Result)
+		}
+	}
+}
+
 func TestProviderValidate(t *testing.T) {
 	cases := []struct {
 		P      *Provider

helper/schema/resource.go

@@ -175,6 +175,33 @@ func (r *Resource) Validate(c *terraform.ResourceConfig) ([]string, []error) {
 	return schemaMap(r.Schema).Validate(c)
 }
 
+// ReadDataApply loads the data for a data source, given a diff that
+// describes the configuration arguments and desired computed attributes.
+func (r *Resource) ReadDataApply(
+	d *terraform.InstanceDiff,
+	meta interface{},
+) (*terraform.InstanceState, error) {
+
+	// Data sources are always built completely from scratch
+	// on each read, so the source state is always nil.
+	data, err := schemaMap(r.Schema).Data(nil, d)
+	if err != nil {
+		return nil, err
+	}
+
+	err = r.Read(data, meta)
+	state := data.State()
+	if state != nil && state.ID == "" {
+		// Data sources can set an ID if they want, but they aren't
+		// required to; we'll provide a placeholder if they don't,
+		// to preserve the invariant that all resources have non-empty
+		// ids.
+		state.ID = "-"
+	}
+
+	return r.recordCurrentSchemaVersion(state), err
+}
+
 // Refresh refreshes the state of the resource.
 func (r *Resource) Refresh(
 	s *terraform.InstanceState,

plugin/resource_provider.go

@@ -156,6 +156,30 @@ func (p *ResourceProvider) Diff(
 	return resp.Diff, err
 }
 
+func (p *ResourceProvider) ValidateDataSource(
+	t string, c *terraform.ResourceConfig) ([]string, []error) {
+	var resp ResourceProviderValidateResourceResponse
+	args := ResourceProviderValidateResourceArgs{
+		Config: c,
+		Type:   t,
+	}
+
+	err := p.Client.Call("Plugin.ValidateDataSource", &args, &resp)
+	if err != nil {
+		return nil, []error{err}
+	}
+
+	var errs []error
+	if len(resp.Errors) > 0 {
+		errs = make([]error, len(resp.Errors))
+		for i, err := range resp.Errors {
+			errs[i] = err
+		}
+	}
+
+	return resp.Warnings, errs
+}
+
 func (p *ResourceProvider) Refresh(
 	info *terraform.InstanceInfo,
 	s *terraform.InstanceState) (*terraform.InstanceState, error) {
@@ -208,6 +232,58 @@ func (p *ResourceProvider) Resources() []terraform.ResourceType {
 	return result
 }
 
+func (p *ResourceProvider) ReadDataDiff(
+	info *terraform.InstanceInfo,
+	c *terraform.ResourceConfig) (*terraform.InstanceDiff, error) {
+	var resp ResourceProviderReadDataDiffResponse
+	args := &ResourceProviderReadDataDiffArgs{
+		Info:   info,
+		Config: c,
+	}
+
+	err := p.Client.Call("Plugin.ReadDataDiff", args, &resp)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		err = resp.Error
+	}
+
+	return resp.Diff, err
+}
+
+func (p *ResourceProvider) ReadDataApply(
+	info *terraform.InstanceInfo,
+	d *terraform.InstanceDiff) (*terraform.InstanceState, error) {
+	var resp ResourceProviderReadDataApplyResponse
+	args := &ResourceProviderReadDataApplyArgs{
+		Info: info,
+		Diff: d,
+	}
+
+	err := p.Client.Call("Plugin.ReadDataApply", args, &resp)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Error != nil {
+		err = resp.Error
+	}
+
+	return resp.State, err
+}
+
+func (p *ResourceProvider) DataSources() []terraform.DataSource {
+	var result []terraform.DataSource
+
+	err := p.Client.Call("Plugin.DataSources", new(interface{}), &result)
+	if err != nil {
+		// TODO: panic, log, what?
+		return nil
+	}
+
+	return result
+}
+
 func (p *ResourceProvider) Close() error {
 	return p.Client.Close()
 }
@@ -275,6 +351,26 @@ type ResourceProviderImportStateResponse struct {
 	Error *plugin.BasicError
 }
 
+type ResourceProviderReadDataApplyArgs struct {
+	Info *terraform.InstanceInfo
+	Diff *terraform.InstanceDiff
+}
+
+type ResourceProviderReadDataApplyResponse struct {
+	State *terraform.InstanceState
+	Error *plugin.BasicError
+}
+
+type ResourceProviderReadDataDiffArgs struct {
+	Info   *terraform.InstanceInfo
+	Config *terraform.ResourceConfig
+}
+
+type ResourceProviderReadDataDiffResponse struct {
+	Diff  *terraform.InstanceDiff
+	Error *plugin.BasicError
+}
+
 type ResourceProviderValidateArgs struct {
 	Config *terraform.ResourceConfig
 }
@@ -408,3 +504,47 @@ func (s *ResourceProviderServer) Resources(
 	*result = s.Provider.Resources()
 	return nil
 }
+
+func (s *ResourceProviderServer) ValidateDataSource(
+	args *ResourceProviderValidateResourceArgs,
+	reply *ResourceProviderValidateResourceResponse) error {
+	warns, errs := s.Provider.ValidateDataSource(args.Type, args.Config)
+	berrs := make([]*plugin.BasicError, len(errs))
+	for i, err := range errs {
+		berrs[i] = plugin.NewBasicError(err)
+	}
+	*reply = ResourceProviderValidateResourceResponse{
+		Warnings: warns,
+		Errors:   berrs,
+	}
+	return nil
+}
+
+func (s *ResourceProviderServer) ReadDataDiff(
+	args *ResourceProviderReadDataDiffArgs,
+	result *ResourceProviderReadDataDiffResponse) error {
+	diff, err := s.Provider.ReadDataDiff(args.Info, args.Config)
+	*result = ResourceProviderReadDataDiffResponse{
+		Diff:  diff,
+		Error: plugin.NewBasicError(err),
+	}
+	return nil
+}
+
+func (s *ResourceProviderServer) ReadDataApply(
+	args *ResourceProviderReadDataApplyArgs,
+	result *ResourceProviderReadDataApplyResponse) error {
+	newState, err := s.Provider.ReadDataApply(args.Info, args.Diff)
+	*result = ResourceProviderReadDataApplyResponse{
+		State: newState,
+		Error: plugin.NewBasicError(err),
+	}
+	return nil
+}
+
+func (s *ResourceProviderServer) DataSources(
+	nothing interface{},
+	result *[]terraform.DataSource) error {
+	*result = s.Provider.DataSources()
+	return nil
+}

plugin/resource_provider_test.go

@@ -389,6 +389,77 @@ func TestResourceProvider_resources(t *testing.T) {
 	}
 }
 
+func TestResourceProvider_readdataapply(t *testing.T) {
+	p := new(terraform.MockResourceProvider)
+
+	// Create a mock provider
+	client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
+		ProviderFunc: testProviderFixed(p),
+	}))
+	defer client.Close()
+
+	// Request the provider
+	raw, err := client.Dispense(ProviderPluginName)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	provider := raw.(terraform.ResourceProvider)
+
+	p.ReadDataApplyReturn = &terraform.InstanceState{
+		ID: "bob",
+	}
+
+	// ReadDataApply
+	info := &terraform.InstanceInfo{}
+	diff := &terraform.InstanceDiff{}
+	newState, err := provider.ReadDataApply(info, diff)
+	if !p.ReadDataApplyCalled {
+		t.Fatal("ReadDataApply should be called")
+	}
+	if !reflect.DeepEqual(p.ReadDataApplyDiff, diff) {
+		t.Fatalf("bad: %#v", p.ReadDataApplyDiff)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+	if !reflect.DeepEqual(p.ReadDataApplyReturn, newState) {
+		t.Fatalf("bad: %#v", newState)
+	}
+}
+
+func TestResourceProvider_datasources(t *testing.T) {
+	p := new(terraform.MockResourceProvider)
+
+	// Create a mock provider
+	client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
+		ProviderFunc: testProviderFixed(p),
+	}))
+	defer client.Close()
+
+	// Request the provider
+	raw, err := client.Dispense(ProviderPluginName)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	provider := raw.(terraform.ResourceProvider)
+
+	expected := []terraform.DataSource{
+		{"foo"},
+		{"bar"},
+	}
+
+	p.DataSourcesReturn = expected
+
+	// DataSources
+	result := provider.DataSources()
+	if !p.DataSourcesCalled {
+		t.Fatal("DataSources should be called")
+	}
+	if !reflect.DeepEqual(result, expected) {
+		t.Fatalf("bad: %#v", result)
+	}
+}
+
 func TestResourceProvider_validate(t *testing.T) {
 	p := new(terraform.MockResourceProvider)
 
@@ -628,6 +699,44 @@ func TestResourceProvider_validateResource_warns(t *testing.T) {
 	}
 }
 
+func TestResourceProvider_validateDataSource(t *testing.T) {
+	p := new(terraform.MockResourceProvider)
+
+	// Create a mock provider
+	client, _ := plugin.TestPluginRPCConn(t, pluginMap(&ServeOpts{
+		ProviderFunc: testProviderFixed(p),
+	}))
+	defer client.Close()
+
+	// Request the provider
+	raw, err := client.Dispense(ProviderPluginName)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	provider := raw.(terraform.ResourceProvider)
+
+	// Configure
+	config := &terraform.ResourceConfig{
+		Raw: map[string]interface{}{"foo": "bar"},
+	}
+	w, e := provider.ValidateDataSource("foo", config)
+	if !p.ValidateDataSourceCalled {
+		t.Fatal("configure should be called")
+	}
+	if p.ValidateDataSourceType != "foo" {
+		t.Fatalf("bad: %#v", p.ValidateDataSourceType)
+	}
+	if !reflect.DeepEqual(p.ValidateDataSourceConfig, config) {
+		t.Fatalf("bad: %#v", p.ValidateDataSourceConfig)
+	}
+	if w != nil {
+		t.Fatalf("bad: %#v", w)
+	}
+	if e != nil {
+		t.Fatalf("bad: %#v", e)
+	}
+}
+
 func TestResourceProvider_close(t *testing.T) {
 	p := new(terraform.MockResourceProvider)
 

terraform/resource_provider.go

@@ -97,6 +97,35 @@ type ResourceProvider interface {
 	// Each rule is represented by a separate resource in Terraform,
 	// therefore multiple states are returned.
 	ImportState(*InstanceInfo, string) ([]*InstanceState, error)
+
+	/*********************************************************************
+	* Functions related to data resources
+	*********************************************************************/
+
+	// ValidateDataSource is called once at the beginning with the raw
+	// configuration (no interpolation done) and can return a list of warnings
+	// and/or errors.
+	//
+	// This is called once per data source instance.
+	//
+	// This should not assume any of the values in the resource configuration
+	// are valid since it is possible they have to be interpolated still.
+	// The primary use case of this call is to check that the required keys
+	// are set and that the general structure is correct.
+	ValidateDataSource(string, *ResourceConfig) ([]string, []error)
+
+	// DataSources returns all of the available data sources that this
+	// provider implements.
+	DataSources() []DataSource
+
+	// ReadDataDiff produces a diff that represents the state that will
+	// be produced when the given data source is read using a later call
+	// to ReadDataApply.
+	ReadDataDiff(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error)
+
+	// ReadDataApply initializes a data instance using the configuration
+	// in a diff produced by ReadDataDiff.
+	ReadDataApply(*InstanceInfo, *InstanceDiff) (*InstanceState, error)
 }
 
 // ResourceProviderCloser is an interface that providers that can close
@@ -111,6 +140,11 @@ type ResourceType struct {
 	Importable bool   // Whether this resource supports importing
 }
 
+// DataSource is a data source that a resource provider implements.
+type DataSource struct {
+	Name string
+}
+
 // ResourceProviderFactory is a function type that creates a new instance
 // of a resource provider.
 type ResourceProviderFactory func() (ResourceProvider, error)
@@ -123,7 +157,7 @@ func ResourceProviderFactoryFixed(p ResourceProvider) ResourceProviderFactory {
 	}
 }
 
-func ProviderSatisfies(p ResourceProvider, n string) bool {
+func ProviderHasResource(p ResourceProvider, n string) bool {
 	for _, rt := range p.Resources() {
 		if rt.Name == n {
 			return true
@@ -132,3 +166,13 @@ func ProviderSatisfies(p ResourceProvider, n string) bool {
 
 	return false
 }
+
+func ProviderHasDataSource(p ResourceProvider, n string) bool {
+	for _, rt := range p.DataSources() {
+		if rt.Name == n {
+			return true
+		}
+	}
+
+	return false
+}

terraform/resource_provider_mock.go

@@ -44,6 +44,20 @@ type MockResourceProvider struct {
 	RefreshReturnError             error
 	ResourcesCalled                bool
 	ResourcesReturn                []ResourceType
+	ReadDataApplyCalled            bool
+	ReadDataApplyInfo              *InstanceInfo
+	ReadDataApplyDiff              *InstanceDiff
+	ReadDataApplyFn                func(*InstanceInfo, *InstanceDiff) (*InstanceState, error)
+	ReadDataApplyReturn            *InstanceState
+	ReadDataApplyReturnError       error
+	ReadDataDiffCalled             bool
+	ReadDataDiffInfo               *InstanceInfo
+	ReadDataDiffDesired            *ResourceConfig
+	ReadDataDiffFn                 func(*InstanceInfo, *ResourceConfig) (*InstanceDiff, error)
+	ReadDataDiffReturn             *InstanceDiff
+	ReadDataDiffReturnError        error
+	DataSourcesCalled              bool
+	DataSourcesReturn              []DataSource
 	ValidateCalled                 bool
 	ValidateConfig                 *ResourceConfig
 	ValidateFn                     func(*ResourceConfig) ([]string, []error)
@@ -55,6 +69,12 @@ type MockResourceProvider struct {
 	ValidateResourceConfig         *ResourceConfig
 	ValidateResourceReturnWarns    []string
 	ValidateResourceReturnErrors   []error
+	ValidateDataSourceFn           func(string, *ResourceConfig) ([]string, []error)
+	ValidateDataSourceCalled       bool
+	ValidateDataSourceType         string
+	ValidateDataSourceConfig       *ResourceConfig
+	ValidateDataSourceReturnWarns  []string
+	ValidateDataSourceReturnErrors []error
 
 	ImportStateCalled      bool
 	ImportStateInfo        *InstanceInfo
@@ -196,3 +216,59 @@ func (p *MockResourceProvider) ImportState(info *InstanceInfo, id string) ([]*In
 
 	return p.ImportStateReturn, p.ImportStateReturnError
 }
+
+func (p *MockResourceProvider) ValidateDataSource(t string, c *ResourceConfig) ([]string, []error) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.ValidateDataSourceCalled = true
+	p.ValidateDataSourceType = t
+	p.ValidateDataSourceConfig = c
+
+	if p.ValidateDataSourceFn != nil {
+		return p.ValidateDataSourceFn(t, c)
+	}
+
+	return p.ValidateDataSourceReturnWarns, p.ValidateDataSourceReturnErrors
+}
+
+func (p *MockResourceProvider) ReadDataDiff(
+	info *InstanceInfo,
+	desired *ResourceConfig) (*InstanceDiff, error) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.ReadDataDiffCalled = true
+	p.ReadDataDiffInfo = info
+	p.ReadDataDiffDesired = desired
+	if p.ReadDataDiffFn != nil {
+		return p.ReadDataDiffFn(info, desired)
+	}
+
+	return p.ReadDataDiffReturn, p.ReadDataDiffReturnError
+}
+
+func (p *MockResourceProvider) ReadDataApply(
+	info *InstanceInfo,
+	d *InstanceDiff) (*InstanceState, error) {
+	p.Lock()
+	defer p.Unlock()
+
+	p.ReadDataApplyCalled = true
+	p.ReadDataApplyInfo = info
+	p.ReadDataApplyDiff = d
+
+	if p.ReadDataApplyFn != nil {
+		return p.ReadDataApplyFn(info, d)
+	}
+
+	return p.ReadDataApplyReturn, p.ReadDataApplyReturnError
+}
+
+func (p *MockResourceProvider) DataSources() []DataSource {
+	p.Lock()
+	defer p.Unlock()
+
+	p.DataSourcesCalled = true
+	return p.DataSourcesReturn
+}