Re-integrate the "terraform" provider into the main binary

As part of the 0.10 core/provider split we moved this provider, along with
all the others, out into its own repository.

In retrospect, the "terraform" provider doesn't really make sense to be
separated since it's just a thin wrapper around some core code anyway,
and so re-integrating it into core avoids the confusion that results when
Terraform Core and the terraform provider have inconsistent versions of
the backend code and dependencies.

There is no good reason to use a different version of the backend code
in the provider than in core, so this new "internal provider" mechanism
is stricter than the old one: it's not possible to use an external build
of this provider at all, and version constraints for it are rejected as
a result.

This provider is also run in-process rather than in a child process, since
again it's just a very thin wrapper around code that's already running
in Terraform core anyway, and so the process barrier between the two does
not create enough advantage to warrant the additional complexity.

command/e2etest/init_test.go

@@ -50,6 +50,37 @@ func TestInitProviders(t *testing.T) {
 
 }
 
+func TestInitProvidersInternal(t *testing.T) {
+	t.Parallel()
+
+	// This test should _not_ reach out anywhere because the "terraform"
+	// provider is internal to the core terraform binary.
+
+	fixturePath := filepath.Join("test-fixtures", "terraform-provider")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
+	defer tf.Close()
+
+	stdout, stderr, err := tf.Run("init")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	if !strings.Contains(stdout, "Terraform has been successfully initialized!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if strings.Contains(stdout, "Downloading plugin for provider") {
+		// Shouldn't have downloaded anything with this config, because the
+		// provider is built in.
+		t.Errorf("provider download message appeared in output:\n%s", stdout)
+	}
+
+}
+
 func TestInitProviders_pluginCache(t *testing.T) {
 	t.Parallel()
 

command/e2etest/test-fixtures/terraform-provider/main.tf

@@ -0,0 +1,10 @@
+provider "terraform" {
+
+}
+
+data "terraform_remote_state" "test" {
+  backend = "local"
+  config = {
+    path = "nothing.tfstate"
+  }
+}

command/init.go

@@ -310,6 +310,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 	))
 
 	missing := c.missingPlugins(available, requirements)
+	internal := c.internalProviders()
 
 	var errs error
 	if c.getPlugins {
@@ -319,6 +320,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 		}
 
 		for provider, reqd := range missing {
+			if _, isInternal := internal[provider]; isInternal {
+				// Ignore internal providers; they are not eligible for
+				// installation.
+				continue
+			}
+
 			_, err := c.providerInstaller.Get(provider, reqd.Versions)
 
 			if err != nil {
@@ -376,7 +383,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
 	// again. If anything changes, other commands that use providers will
 	// fail with an error instructing the user to re-run this command.
 	available = c.providerPluginSet() // re-discover to see newly-installed plugins
-	chosen := choosePlugins(available, requirements)
+	chosen := choosePlugins(available, internal, requirements)
 	digests := map[string][]byte{}
 	for name, meta := range chosen {
 		digest, err := meta.SHA256()

command/plugins.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 
 	plugin "github.com/hashicorp/go-plugin"
+	terraformProvider "github.com/hashicorp/terraform/builtin/providers/terraform"
 	tfplugin "github.com/hashicorp/terraform/plugin"
 	"github.com/hashicorp/terraform/plugin/discovery"
 	"github.com/hashicorp/terraform/terraform"
@@ -25,12 +26,25 @@ import (
 // each that satisfies the given constraints.
 type multiVersionProviderResolver struct {
 	Available discovery.PluginMetaSet
+
+	// Internal is a map that overrides the usual plugin selection process
+	// for internal plugins. These plugins do not support version constraints
+	// (will produce an error if one is set). This should be used only in
+	// exceptional circumstances since it forces the provider's release
+	// schedule to be tied to that of Terraform Core.
+	Internal map[string]terraform.ResourceProviderFactory
 }
 
-func choosePlugins(avail discovery.PluginMetaSet, reqd discovery.PluginRequirements) map[string]discovery.PluginMeta {
+func choosePlugins(avail discovery.PluginMetaSet, internal map[string]terraform.ResourceProviderFactory, reqd discovery.PluginRequirements) map[string]discovery.PluginMeta {
 	candidates := avail.ConstrainVersions(reqd)
 	ret := map[string]discovery.PluginMeta{}
 	for name, metas := range candidates {
+		// If the provider is in our internal map then we ignore any
+		// discovered plugins for it since these are dealt with separately.
+		if _, isInternal := internal[name]; isInternal {
+			continue
+		}
+
 		if len(metas) == 0 {
 			continue
 		}
@@ -45,8 +59,17 @@ func (r *multiVersionProviderResolver) ResolveProviders(
 	factories := make(map[string]terraform.ResourceProviderFactory, len(reqd))
 	var errs []error
 
-	chosen := choosePlugins(r.Available, reqd)
+	chosen := choosePlugins(r.Available, r.Internal, reqd)
 	for name, req := range reqd {
+		if factory, isInternal := r.Internal[name]; isInternal {
+			if !req.Versions.Unconstrained() {
+				errs = append(errs, fmt.Errorf("provider.%s: this provider is built in to Terraform and so it does not support version constraints", name))
+				continue
+			}
+			factories[name] = factory
+			continue
+		}
+
 		if newest, available := chosen[name]; available {
 			digest, err := newest.SHA256()
 			if err != nil {
@@ -233,6 +256,15 @@ func (m *Meta) providerPluginManuallyInstalledSet() discovery.PluginMetaSet {
 func (m *Meta) providerResolver() terraform.ResourceProviderResolver {
 	return &multiVersionProviderResolver{
 		Available: m.providerPluginSet(),
+		Internal:  m.internalProviders(),
+	}
+}
+
+func (m *Meta) internalProviders() map[string]terraform.ResourceProviderFactory {
+	return map[string]terraform.ResourceProviderFactory{
+		"terraform": func() (terraform.ResourceProvider, error) {
+			return terraformProvider.Provider(), nil
+		},
 	}
 }
 

command/plugins_test.go

@@ -9,8 +9,91 @@ import (
 	"testing"
 
 	"github.com/hashicorp/terraform/plugin/discovery"
+	"github.com/hashicorp/terraform/terraform"
 )
 
+func TestMultiVersionProviderResolver(t *testing.T) {
+	available := make(discovery.PluginMetaSet)
+	available.Add(discovery.PluginMeta{
+		Name:    "plugin",
+		Version: "1.0.0",
+		Path:    "test-fixtures/empty-file",
+	})
+
+	resolver := &multiVersionProviderResolver{
+		Internal: map[string]terraform.ResourceProviderFactory{
+			"internal": func() (terraform.ResourceProvider, error) {
+				return &terraform.MockResourceProvider{
+					ResourcesReturn: []terraform.ResourceType{
+						{
+							Name: "internal_foo",
+						},
+					},
+				}, nil
+			},
+		},
+		Available: available,
+	}
+
+	t.Run("plugin matches", func(t *testing.T) {
+		reqd := discovery.PluginRequirements{
+			"plugin": &discovery.PluginConstraints{
+				Versions: discovery.ConstraintStr("1.0.0").MustParse(),
+			},
+		}
+		got, err := resolver.ResolveProviders(reqd)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if ct := len(got); ct != 1 {
+			t.Errorf("wrong number of results %d; want 1", ct)
+		}
+		if _, exists := got["plugin"]; !exists {
+			t.Errorf("provider \"plugin\" not in result")
+		}
+	})
+	t.Run("plugin doesn't match", func(t *testing.T) {
+		reqd := discovery.PluginRequirements{
+			"plugin": &discovery.PluginConstraints{
+				Versions: discovery.ConstraintStr("2.0.0").MustParse(),
+			},
+		}
+		_, err := resolver.ResolveProviders(reqd)
+		if err == nil {
+			t.Errorf("resolved successfully, but want error")
+		}
+	})
+	t.Run("internal matches", func(t *testing.T) {
+		reqd := discovery.PluginRequirements{
+			"internal": &discovery.PluginConstraints{
+				Versions: discovery.AllVersions,
+			},
+		}
+		got, err := resolver.ResolveProviders(reqd)
+		if err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+		if ct := len(got); ct != 1 {
+			t.Errorf("wrong number of results %d; want 1", ct)
+		}
+		if _, exists := got["internal"]; !exists {
+			t.Errorf("provider \"internal\" not in result")
+		}
+	})
+	t.Run("internal with version constraint", func(t *testing.T) {
+		// Version constraints are not permitted for internal providers
+		reqd := discovery.PluginRequirements{
+			"internal": &discovery.PluginConstraints{
+				Versions: discovery.ConstraintStr("2.0.0").MustParse(),
+			},
+		}
+		_, err := resolver.ResolveProviders(reqd)
+		if err == nil {
+			t.Errorf("resolved successfully, but want error")
+		}
+	})
+}
+
 func TestPluginPath(t *testing.T) {
 	td, err := ioutil.TempDir("", "tf")
 	if err != nil {
@@ -36,6 +119,26 @@ func TestPluginPath(t *testing.T) {
 	}
 }
 
+func TestInternalProviders(t *testing.T) {
+	m := Meta{}
+	internal := m.internalProviders()
+	tfProvider, err := internal["terraform"]()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dataSources := tfProvider.DataSources()
+	found := false
+	for _, ds := range dataSources {
+		if ds.Name == "terraform_remote_state" {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("didn't find terraform_remote_state in internal \"terraform\" provider")
+	}
+}
+
 // mockProviderInstaller is a discovery.PluginInstaller implementation that
 // is a mock for discovery.ProviderInstaller.
 type mockProviderInstaller struct {