core: Attach resource and provider config schemas during graph build

This is a little awkward since we need to instantiate the providers much
earlier than before. To avoid a lot of reshuffling here we just spin each
one up and then immediately shut it down again, letting our existing init
functionality during the graph walk still do the main initialization.

terraform/context.go

@@ -249,14 +249,13 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags.
 	switch typ {
 	case GraphTypeApply:
 		return (&ApplyGraphBuilder{
-			Config:       c.config,
+			Config:     c.config,
-			Diff:         c.diff,
+			Diff:       c.diff,
-			State:        c.state,
+			State:      c.state,
-			Providers:    c.components.ResourceProviders(),
+			Components: c.components,
-			Provisioners: c.components.ResourceProvisioners(),
+			Targets:    c.targets,
-			Targets:      c.targets,
+			Destroy:    c.destroy,
-			Destroy:      c.destroy,
+			Validate:   opts.Validate,
-			Validate:     opts.Validate,
 		}).Build(addrs.RootModuleInstance)
 
 	case GraphTypeInput:
@@ -268,11 +267,11 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags.
 	case GraphTypePlan:
 		// Create the plan graph builder
 		p := &PlanGraphBuilder{
-			Config:    c.config,
+			Config:     c.config,
-			State:     c.state,
+			State:      c.state,
-			Providers: c.components.ResourceProviders(),
+			Components: c.components,
-			Targets:   c.targets,
+			Targets:    c.targets,
-			Validate:  opts.Validate,
+			Validate:   opts.Validate,
 		}
 
 		// Some special cases for other graph types shared with plan currently
@@ -281,9 +280,6 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags.
 		case GraphTypeInput:
 			b = InputGraphBuilder(p)
 		case GraphTypeValidate:
-			// We need to set the provisioners so those can be validated
-			p.Provisioners = c.components.ResourceProvisioners()
-
 			b = ValidateGraphBuilder(p)
 		}
 
@@ -299,11 +295,11 @@ func (c *Context) Graph(typ GraphType, opts *ContextGraphOpts) (*Graph, tfdiags.
 
 	case GraphTypeRefresh:
 		return (&RefreshGraphBuilder{
-			Config:    c.config,
+			Config:     c.config,
-			State:     c.state,
+			State:      c.state,
-			Providers: c.components.ResourceProviders(),
+			Components: c.components,
-			Targets:   c.targets,
+			Targets:    c.targets,
-			Validate:  opts.Validate,
+			Validate:   opts.Validate,
 		}).Build(addrs.RootModuleInstance)
 
 	default:

terraform/graph_builder_apply.go

@@ -24,11 +24,9 @@ type ApplyGraphBuilder struct {
 	// State is the current state
 	State *State
 
-	// Providers is the list of providers supported.
+	// Components is a factory for the plug-in components (providers and
-	Providers []string
+	// provisioners) available for use.
-
+	Components contextComponentFactory
-	// Provisioners is the list of provisioners supported.
-	Provisioners []string
 
 	// Targets are resources to target. This is only required to make sure
 	// unnecessary outputs aren't included in the apply graph. The plan
@@ -87,7 +85,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
 		&AttachStateTransformer{State: b.State},
 
 		// add providers
-		TransformProviders(b.Providers, concreteProvider, b.Config),
+		TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config),
 
 		// Destruction ordering
 		&DestroyEdgeTransformer{Config: b.Config, State: b.State},
@@ -97,7 +95,7 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
 		),
 
 		// Provisioner-related transformations
-		&MissingProvisionerTransformer{Provisioners: b.Provisioners},
+		&MissingProvisionerTransformer{Provisioners: b.Components.ResourceProvisioners()},
 		&ProvisionerTransformer{},
 
 		// Add root variables
@@ -115,6 +113,10 @@ func (b *ApplyGraphBuilder) Steps() []GraphTransformer {
 		// Remove modules no longer present in the config
 		&RemovedModuleTransformer{Config: b.Config, State: b.State},
 
+		// Must be before ReferenceTransformer, since schema is required to
+		// extract references from config.
+		&AttachSchemaTransformer{Components: b.Components},
+
 		// Connect references so ordering is correct
 		&ReferenceTransformer{},
 

terraform/graph_builder_plan.go

@@ -29,11 +29,9 @@ type PlanGraphBuilder struct {
 	// State is the current state
 	State *State
 
-	// Providers is the list of providers supported.
+	// Components is a factory for the plug-in components (providers and
-	Providers []string
+	// provisioners) available for use.
-
+	Components contextComponentFactory
-	// Provisioners is the list of provisioners supported.
-	Provisioners []string
 
 	// Targets are resources to target
 	Targets []addrs.Targetable
@@ -103,16 +101,11 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer {
 		// Add root variables
 		&RootVariableTransformer{Config: b.Config},
 
-		TransformProviders(b.Providers, b.ConcreteProvider, b.Config),
+		TransformProviders(b.Components.ResourceProviders(), b.ConcreteProvider, b.Config),
 
-		// Provisioner-related transformations. Only add these if requested.
+		&MissingProvisionerTransformer{Provisioners: b.Components.ResourceProvisioners()},
-		GraphTransformIf(
+
-			func() bool { return b.Provisioners != nil },
+		&AttachSchemaTransformer{Components: b.Components},
-			GraphTransformMulti(
-				&MissingProvisionerTransformer{Provisioners: b.Provisioners},
-				&ProvisionerTransformer{},
-			),
-		),
 
 		// Add module variables
 		&ModuleVariableTransformer{
@@ -122,6 +115,10 @@ func (b *PlanGraphBuilder) Steps() []GraphTransformer {
 		// Remove modules no longer present in the config
 		&RemovedModuleTransformer{Config: b.Config, State: b.State},
 
+		// Must be before ReferenceTransformer, since schema is required to
+		// extract references from config.
+		&AttachSchemaTransformer{Components: b.Components},
+
 		// Connect so that the references are ready for targeting. We'll
 		// have to connect again later for providers and so on.
 		&ReferenceTransformer{},

terraform/graph_builder_refresh.go

@@ -29,8 +29,9 @@ type RefreshGraphBuilder struct {
 	// State is the current state
 	State *State
 
-	// Providers is the list of providers supported.
+	// Components is a factory for the plug-in components (providers and
-	Providers []string
+	// provisioners) available for use.
+	Components contextComponentFactory
 
 	// Targets are resources to target
 	Targets []addrs.Targetable
@@ -124,7 +125,7 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer {
 		// Add root variables
 		&RootVariableTransformer{Config: b.Config},
 
-		TransformProviders(b.Providers, concreteProvider, b.Config),
+		TransformProviders(b.Components.ResourceProviders(), concreteProvider, b.Config),
 
 		// Add the local values
 		&LocalTransformer{Config: b.Config},
@@ -135,6 +136,10 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer {
 		// Add module variables
 		&ModuleVariableTransformer{Config: b.Config},
 
+		// Must be before ReferenceTransformer, since schema is required to
+		// extract references from config.
+		&AttachSchemaTransformer{Components: b.Components},
+
 		// Connect so that the references are ready for targeting. We'll
 		// have to connect again later for providers and so on.
 		&ReferenceTransformer{},

terraform/graph_walk_context.go

@@ -73,7 +73,18 @@ func (w *ContextGraphWalker) EnterPath(path addrs.ModuleInstance) EvalContext {
 	// so that we can safely run multiple evaluations at once across
 	// different modules.
 	evaluator := &Evaluator{
-		StateLock: &w.Context.stateLock,
+		Meta:            w.Context.meta,
+		Config:          w.Context.config,
+		State:           w.Context.state,
+		StateLock:       &w.Context.stateLock,
+		ProviderSchemas: w.providerSchemas,
+		ProvidersLock:   &w.providerLock,
+
+		// FIXME: This was a design mistake on the evaluator, which should
+		// get replaced with something like the interpolatorVars thing above
+		// once we verify exactly how that was used in the old Interpolator
+		// codepath.
+		RootVariableValues: map[string]*InputValue{},
 	}
 
 	ctx := &BuiltinEvalContext{
@@ -84,6 +95,7 @@ func (w *ContextGraphWalker) EnterPath(path addrs.ModuleInstance) EvalContext {
 		Components:          w.Context.components,
 		ProviderCache:       w.providerCache,
 		ProviderInputConfig: w.Context.providerInputConfig,
+		ProviderSchemas:     w.providerSchemas,
 		ProviderLock:        &w.providerLock,
 		ProvisionerCache:    w.provisionerCache,
 		ProvisionerLock:     &w.provisionerLock,

terraform/node_data_refresh.go

@@ -66,6 +66,7 @@ func (n *NodeRefreshableDataResource) DynamicExpand(ctx EvalContext) (*Graph, er
 		// Expand the count.
 		&ResourceCountTransformer{
 			Concrete: concreteResource,
+			Schema:   n.Schema,
 			Count:    count,
 			Addr:     n.ResourceAddr(),
 		},

terraform/node_provider_abstract.go

@@ -2,6 +2,7 @@ package terraform
 
 import (
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/config/configschema"
 	"github.com/hashicorp/terraform/configs"
 
 	"github.com/hashicorp/terraform/dag"
@@ -21,16 +22,17 @@ type NodeAbstractProvider struct {
 	// set if you already have that information.
 
 	Config *configs.Provider
-	Schema *ProviderSchema
+	Schema *configschema.Block
 }
 
 var (
-	_ GraphNodeSubPath        = (*NodeAbstractProvider)(nil)
+	_ GraphNodeSubPath                    = (*NodeAbstractProvider)(nil)
-	_ RemovableIfNotTargeted  = (*NodeAbstractProvider)(nil)
+	_ RemovableIfNotTargeted              = (*NodeAbstractProvider)(nil)
-	_ GraphNodeReferencer     = (*NodeAbstractProvider)(nil)
+	_ GraphNodeReferencer                 = (*NodeAbstractProvider)(nil)
-	_ GraphNodeProvider       = (*NodeAbstractProvider)(nil)
+	_ GraphNodeProvider                   = (*NodeAbstractProvider)(nil)
-	_ GraphNodeAttachProvider = (*NodeAbstractProvider)(nil)
+	_ GraphNodeAttachProvider             = (*NodeAbstractProvider)(nil)
-	_ dag.GraphNodeDotter     = (*NodeAbstractProvider)(nil)
+	_ GraphNodeAttachProviderConfigSchema = (*NodeAbstractProvider)(nil)
+	_ dag.GraphNodeDotter                 = (*NodeAbstractProvider)(nil)
 )
 
 func (n *NodeAbstractProvider) Name() string {
@@ -55,7 +57,7 @@ func (n *NodeAbstractProvider) References() []*addrs.Reference {
 		return nil
 	}
 
-	return ReferencesFromConfig(n.Config.Config, n.Schema.Provider)
+	return ReferencesFromConfig(n.Config.Config, n.Schema)
 }
 
 // GraphNodeProvider
@@ -77,9 +79,9 @@ func (n *NodeAbstractProvider) AttachProvider(c *configs.Provider) {
 	n.Config = c
 }
 
-// GraphNodeAttachProvider
+// GraphNodeAttachProviderConfigSchema impl.
-func (n *NodeAbstractProvider) AttachProviderSchema(s *ProviderSchema) {
+func (n *NodeAbstractProvider) AttachProviderConfigSchema(schema *configschema.Block) {
-	n.Schema = s
+	n.Schema = schema
 }
 
 // GraphNodeDotter impl.

terraform/node_resource_abstract.go

@@ -68,6 +68,7 @@ var (
 	_ GraphNodeProvisionerConsumer  = (*NodeAbstractResource)(nil)
 	_ GraphNodeResource             = (*NodeAbstractResource)(nil)
 	_ GraphNodeAttachResourceConfig = (*NodeAbstractResource)(nil)
+	_ GraphNodeAttachResourceSchema = (*NodeAbstractResource)(nil)
 	_ GraphNodeTargetable           = (*NodeAbstractResource)(nil)
 	_ dag.GraphNodeDotter           = (*NodeAbstractResource)(nil)
 )
@@ -106,7 +107,8 @@ var (
 	_ GraphNodeResourceInstance     = (*NodeAbstractResourceInstance)(nil)
 	_ GraphNodeAttachResourceState  = (*NodeAbstractResourceInstance)(nil)
 	_ GraphNodeAttachResourceConfig = (*NodeAbstractResourceInstance)(nil)
-	_ GraphNodeTargetable           = (*NodeAbstractResource)(nil)
+	_ GraphNodeAttachResourceSchema = (*NodeAbstractResourceInstance)(nil)
+	_ GraphNodeTargetable           = (*NodeAbstractResourceInstance)(nil)
 	_ dag.GraphNodeDotter           = (*NodeAbstractResourceInstance)(nil)
 )
 
@@ -176,6 +178,12 @@ func (n *NodeAbstractResource) References() []*addrs.Reference {
 			result = append(result, ref)
 		}
 
+		if n.Schema == nil {
+			// Should never happens, but we'll log if it does so that we can
+			// see this easily when debugging.
+			log.Printf("[WARN] no schema is attached to %s, so references cannot be detected", n.Name())
+		}
+
 		refs, _ := lang.ReferencesInExpr(c.Count)
 		result = append(result, refs...)
 		refs, _ = lang.ReferencesInBlock(c.Config, n.Schema)
@@ -353,7 +361,7 @@ func (n *NodeAbstractResource) ProvisionedBy() []string {
 }
 
 // GraphNodeProvisionerConsumer
-func (n *NodeAbstractResource) SetProvisionerSchema(name string, schema *configschema.Block) {
+func (n *NodeAbstractResource) AttachProvisionerSchema(name string, schema *configschema.Block) {
 	n.ProvisionerSchemas[name] = schema
 }
 
@@ -387,6 +395,11 @@ func (n *NodeAbstractResource) AttachResourceConfig(c *configs.Resource) {
 	n.Config = c
 }
 
+// GraphNodeAttachResourceSchema impl
+func (n *NodeAbstractResource) AttachResourceSchema(schema *configschema.Block) {
+	n.Schema = schema
+}
+
 // GraphNodeDotter impl.
 func (n *NodeAbstractResource) DotNode(name string, opts *dag.DotOpts) *dag.DotNode {
 	return &dag.DotNode{

terraform/node_resource_plan.go

@@ -66,6 +66,7 @@ func (n *NodePlannableResource) DynamicExpand(ctx EvalContext) (*Graph, error) {
 		// Expand the count.
 		&ResourceCountTransformer{
 			Concrete: concreteResource,
+			Schema:   n.Schema,
 			Count:    count,
 			Addr:     n.ResourceAddr(),
 		},

terraform/node_resource_refresh.go

@@ -58,6 +58,7 @@ func (n *NodeRefreshableManagedResource) DynamicExpand(ctx EvalContext) (*Graph,
 		// Expand the count.
 		&ResourceCountTransformer{
 			Concrete: concreteResource,
+			Schema:   n.Schema,
 			Count:    count,
 			Addr:     n.ResourceAddr(),
 		},

terraform/node_resource_validate.go

@@ -65,6 +65,7 @@ func (n *NodeValidatableResource) DynamicExpand(ctx EvalContext) (*Graph, error)
 		// Expand the count.
 		&ResourceCountTransformer{
 			Concrete: concreteResource,
+			Schema:   n.Schema,
 			Count:    count,
 			Addr:     n.ResourceAddr(),
 		},

terraform/transform_attach_config_provider.go

@@ -16,7 +16,4 @@ type GraphNodeAttachProvider interface {
 
 	// Sets the configuration
 	AttachProvider(*configs.Provider)
-
-	// Sets the schema
-	AttachProviderSchema(*ProviderSchema)
 }

terraform/transform_attach_schema.go

@@ -0,0 +1,132 @@
+package terraform
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/hashicorp/terraform/dag"
+
+	"github.com/hashicorp/terraform/addrs"
+
+	"github.com/hashicorp/terraform/config/configschema"
+)
+
+// GraphNodeAttachResourceSchema is an interface implemented by node types
+// that need a resource schema attached.
+type GraphNodeAttachResourceSchema interface {
+	GraphNodeResource
+	GraphNodeProviderConsumer
+
+	AttachResourceSchema(*configschema.Block)
+}
+
+// GraphNodeAttachProviderConfigSchema is an interface implemented by node types
+// that need a provider configuration schema attached.
+type GraphNodeAttachProviderConfigSchema interface {
+	GraphNodeProvider
+
+	AttachProviderConfigSchema(*configschema.Block)
+}
+
+// AttachSchemaTransformer finds nodes that implement either
+// GraphNodeAttachResourceSchema or GraphNodeAttachProviderConfigSchema, looks up
+// the schema for each, and then passes it to a method implemented by the
+// node.
+type AttachSchemaTransformer struct {
+	Components contextComponentFactory
+}
+
+func (t *AttachSchemaTransformer) Transform(g *Graph) error {
+
+	// First we'll figure out which provider types we need to fetch schemas for.
+	needProviders := make(map[string]struct{})
+	for _, v := range g.Vertices() {
+		switch tv := v.(type) {
+		case GraphNodeAttachResourceSchema:
+			providerAddr, _ := tv.ProvidedBy()
+			needProviders[providerAddr.ProviderConfig.Type] = struct{}{}
+		case GraphNodeAttachProviderConfigSchema:
+			providerAddr := tv.ProviderAddr()
+			needProviders[providerAddr.ProviderConfig.Type] = struct{}{}
+		}
+	}
+
+	// Now we'll fetch each one. This requires us to temporarily instantiate
+	// them, though this is not a full bootstrap since we don't yet have
+	// configuration information; the providers will be re-instantiated and
+	// properly configured during the graph walk.
+	schemas := make(map[string]*ProviderSchema)
+	for typeName := range needProviders {
+		log.Printf("[TRACE] AttachSchemaTransformer: retrieving schema for provider type %q", typeName)
+		provider, err := t.Components.ResourceProvider(typeName, "early/"+typeName)
+		if err != nil {
+			return fmt.Errorf("failed to instantiate provider %q to obtain schema: %s", typeName, err)
+		}
+
+		// FIXME: The provider interface is currently awkward in that it
+		// requires us to tell the provider which resources types and data
+		// sources we need. In future this will change to just return
+		// everything available, but for now we'll fake that by fetching all
+		// of the available names and then requesting them.
+		resourceTypes := provider.Resources()
+		dataSources := provider.DataSources()
+		resourceTypeNames := make([]string, len(resourceTypes))
+		for i, o := range resourceTypes {
+			resourceTypeNames[i] = o.Name
+		}
+		dataSourceNames := make([]string, len(dataSources))
+		for i, o := range dataSources {
+			dataSourceNames[i] = o.Name
+		}
+
+		schema, err := provider.GetSchema(&ProviderSchemaRequest{
+			ResourceTypes: resourceTypeNames,
+			DataSources:   dataSourceNames,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to retrieve schema from provider %q: %s", typeName, err)
+		}
+
+		schemas[typeName] = schema
+
+		if closer, ok := provider.(ResourceProviderCloser); ok {
+			closer.Close()
+		}
+	}
+
+	// Finally we'll once again visit all of the vertices and attach to
+	// them the schemas we found for them.
+	for _, v := range g.Vertices() {
+		switch tv := v.(type) {
+		case GraphNodeAttachResourceSchema:
+			addr := tv.ResourceAddr()
+			mode := addr.Resource.Mode
+			typeName := addr.Resource.Type
+			providerAddr, _ := tv.ProvidedBy()
+			var schema *configschema.Block
+			switch mode {
+			case addrs.ManagedResourceMode:
+				schema = schemas[providerAddr.ProviderConfig.Type].ResourceTypes[typeName]
+			case addrs.DataResourceMode:
+				schema = schemas[providerAddr.ProviderConfig.Type].DataSources[typeName]
+			}
+			if schema != nil {
+				log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v))
+				tv.AttachResourceSchema(schema)
+			} else {
+				log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", addr)
+			}
+		case GraphNodeAttachProviderConfigSchema:
+			providerAddr := tv.ProviderAddr()
+			schema := schemas[providerAddr.ProviderConfig.Type].Provider
+			if schema != nil {
+				log.Printf("[TRACE] AttachSchemaTransformer: attaching schema to %s", dag.VertexName(v))
+				tv.AttachProviderConfigSchema(schema)
+			} else {
+				log.Printf("[ERROR] AttachSchemaTransformer: No schema available for %s", providerAddr)
+			}
+		}
+	}
+
+	return nil
+}

terraform/transform_provisioner.go

@@ -34,7 +34,7 @@ type GraphNodeProvisionerConsumer interface {
 	// type returned from ProvisionedBy, providing the configuration schema
 	// for each provisioner in turn. The implementer should save these for
 	// later use in evaluating provisioner configuration blocks.
-	SetProvisionerSchema(name string, schema *configschema.Block)
+	AttachProvisionerSchema(name string, schema *configschema.Block)
 }
 
 // ProvisionerTransformer is a GraphTransformer that maps resources to

terraform/transform_resource_count.go

@@ -2,6 +2,7 @@ package terraform
 
 import (
 	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/config/configschema"
 	"github.com/hashicorp/terraform/dag"
 )
 
@@ -11,6 +12,7 @@ import (
 // This assumes that the count is already interpolated.
 type ResourceCountTransformer struct {
 	Concrete ConcreteResourceInstanceNodeFunc
+	Schema   *configschema.Block
 
 	// Count is either the number of indexed instances to create, or -1 to
 	// indicate that count is not set at all and thus a no-key instance should
@@ -25,6 +27,7 @@ func (t *ResourceCountTransformer) Transform(g *Graph) error {
 		addr := t.Addr.Instance(addrs.NoKey)
 
 		abstract := NewNodeAbstractResourceInstance(addr)
+		abstract.Schema = t.Schema
 		var node dag.Vertex = abstract
 		if f := t.Concrete; f != nil {
 			node = f(abstract)