core: New refresh graph building behaviour

Currently, the refresh graph uses the resources from state as a base,
with data sources then layered on. Config is not consulted for resources
and hence new resources that are added with count (or any new resource
from config, for that matter) do not get added to the graph during
refresh.

This is leading to issues with scale in and scale out when the same
value for count is used in both resources, and data sources that may
depend on that resource (and possibly vice versa). While the resources
exist in config and can be used, the fact that ConfigTransformer for
resources is missing means that they don't get added into the graph,
leading to "index out of range" errors and what not.

Further to that, if we add these new resources to the graph for scale
out, considerations need to be taken for scale in as well, which are not
being caught 100% by the current implementation of
NodeRefreshableDataResource. Scale-in resources should be treated as
orphans, which according to the instance-form NodeRefreshableResource
node, should be NodeDestroyableDataResource nodes, but this this logic
is currently not rolled into NodeRefreshableDataResource. This causes
issues on scale-in in the form of race-ish "index out of range" errors
again.

This commit updates the refresh graph so that StateTransformer is no
longer used as the base of the graph. Instead, we add resources from the
state and config in a hybrid fashion:

 * First off, resource nodes are added from config, but only if
   resources currently exist in state.  NodeRefreshableManagedResource
   is a new expandable resource node that will expand count and add
   orphans from state. Any count-expanded node that has config but no
   state is also transformed into a plannable resource, via a new
   ResourceRefreshPlannableTransformer.
 * The NodeRefreshableDataResource node type will now add count orphans
   as NodeDestroyableDataResource nodes. This achieves the same effect
   as if the data sources were added by StateTransformer, but ensures
   there are no races in the dependency chain, with the added benefit of
   directing these nodes straight to the proper
   NodeDestroyableDataResource node.
 * Finally, config orphans (nodes that don't exist in config anymore
   period) are then added, to complete the graph.

This should ensure as much as possible that there is a refresh graph
that best represents both the current state and config with updated
variables and counts.

terraform/graph_builder_refresh.go

@@ -1,6 +1,8 @@
 package terraform
 
 import (
+	"log"
+
 	"github.com/hashicorp/terraform/config"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/dag"
@@ -56,8 +58,16 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer {
 		}
 	}
 
-	concreteResource := func(a *NodeAbstractResource) dag.Vertex {
-		return &NodeRefreshableResourceInstance{
+	concreteManagedResource := func(a *NodeAbstractResource) dag.Vertex {
+		return &NodeRefreshableManagedResource{
+			NodeAbstractCountResource: &NodeAbstractCountResource{
+				NodeAbstractResource: a,
+			},
+		}
+	}
+
+	concreteManagedResourceInstance := func(a *NodeAbstractResource) dag.Vertex {
+		return &NodeRefreshableManagedResourceInstance{
 			NodeAbstractResource: a,
 		}
 	}
@@ -71,13 +81,25 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer {
 	}
 
 	steps := []GraphTransformer{
-		// Creates all the resources represented in the state
-		&StateTransformer{
-			Concrete: concreteResource,
-			State:    b.State,
-		},
+		// Creates all the managed resources that aren't in the state, but only if
+		// we have a state already. No resources in state means there's not
+		// anything to refresh.
+		func() GraphTransformer {
+			if b.State.HasResources() {
+				return &ConfigTransformer{
+					Concrete:   concreteManagedResource,
+					Module:     b.Module,
+					Unique:     true,
+					ModeFilter: true,
+					Mode:       config.ManagedResourceMode,
+				}
+			}
+			log.Println("[TRACE] No managed resources in state during refresh, skipping managed resource transformer")
+			return nil
+		}(),
 
-		// Creates all the data resources that aren't in the state
+		// Creates all the data resources that aren't in the state. This will also
+		// add any orphans from scaling in as destroy nodes.
 		&ConfigTransformer{
 			Concrete:   concreteDataResource,
 			Module:     b.Module,
@@ -86,6 +108,15 @@ func (b *RefreshGraphBuilder) Steps() []GraphTransformer {
 			Mode:       config.DataResourceMode,
 		},
 
+		// Add any fully-orphaned resources from config (ones that have been
+		// removed completely, not ones that are just orphaned due to a scaled-in
+		// count.
+		&OrphanResourceTransformer{
+			Concrete: concreteManagedResourceInstance,
+			State:    b.State,
+			Module:   b.Module,
+		},
+
 		// Attach the state
 		&AttachStateTransformer{State: b.State},
 

terraform/node_data_refresh.go

@@ -33,6 +33,17 @@ func (n *NodeRefreshableDataResource) DynamicExpand(ctx EvalContext) (*Graph, er
 		}
 	}
 
+	// We also need a destroyable resource for orphans that are a result of a
+	// scaled-in count.
+	concreteResourceDestroyable := func(a *NodeAbstractResource) dag.Vertex {
+		// Add the config since we don't do that via transforms
+		a.Config = n.Config
+
+		return &NodeDestroyableDataResource{
+			NodeAbstractResource: n.NodeAbstractResource,
+		}
+	}
+
 	// Start creating the steps
 	steps := []GraphTransformer{
 		// Expand the count.
@@ -42,6 +53,15 @@ func (n *NodeRefreshableDataResource) DynamicExpand(ctx EvalContext) (*Graph, er
 			Addr:     n.ResourceAddr(),
 		},
 
+		// Add the count orphans. As these are orphaned refresh nodes, we add them
+		// directly as NodeDestroyableDataResource.
+		&OrphanResourceCountTransformer{
+			Concrete: concreteResourceDestroyable,
+			Count:    count,
+			Addr:     n.ResourceAddr(),
+			State:    state,
+		},
+
 		// Attach the state
 		&AttachStateTransformer{State: state},
 

terraform/node_resource_refresh.go

@@ -4,21 +4,99 @@ import (
 	"fmt"
 
 	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/dag"
 )
 
-// NodeRefreshableResourceInstance represents a resource that is "applyable":
+// NodeRefreshableManagedResource represents a resource that is expanabled into
+// NodeRefreshableManagedResourceInstance. Resource count orphans are also added.
+type NodeRefreshableManagedResource struct {
+	*NodeAbstractCountResource
+}
+
+// GraphNodeDynamicExpandable
+func (n *NodeRefreshableManagedResource) DynamicExpand(ctx EvalContext) (*Graph, error) {
+	// Grab the state which we read
+	state, lock := ctx.State()
+	lock.RLock()
+	defer lock.RUnlock()
+
+	// Expand the resource count which must be available by now from EvalTree
+	count, err := n.Config.Count()
+	if err != nil {
+		return nil, err
+	}
+
+	// The concrete resource factory we'll use
+	concreteResource := func(a *NodeAbstractResource) dag.Vertex {
+		// Add the config and state since we don't do that via transforms
+		a.Config = n.Config
+
+		return &NodeRefreshableManagedResourceInstance{
+			NodeAbstractResource: a,
+		}
+	}
+
+	// Start creating the steps
+	steps := []GraphTransformer{
+		// Expand the count.
+		&ResourceCountTransformer{
+			Concrete: concreteResource,
+			Count:    count,
+			Addr:     n.ResourceAddr(),
+		},
+
+		// Switch up any node missing state to a plannable resource. This helps
+		// catch cases where data sources depend on the counts from this resource
+		// during a scale out.
+		&ResourceRefreshPlannableTransformer{
+			State: state,
+		},
+
+		// Add the count orphans to make sure these resources are accounted for
+		// during a scale in.
+		&OrphanResourceCountTransformer{
+			Concrete: concreteResource,
+			Count:    count,
+			Addr:     n.ResourceAddr(),
+			State:    state,
+		},
+
+		// Attach the state
+		&AttachStateTransformer{State: state},
+
+		// Targeting
+		&TargetsTransformer{ParsedTargets: n.Targets},
+
+		// Connect references so ordering is correct
+		&ReferenceTransformer{},
+
+		// Make sure there is a single root
+		&RootTransformer{},
+	}
+
+	// Build the graph
+	b := &BasicGraphBuilder{
+		Steps:    steps,
+		Validate: true,
+		Name:     "NodeRefreshableManagedResource",
+	}
+
+	return b.Build(ctx.Path())
+}
+
+// NodeRefreshableManagedResourceInstance represents a resource that is "applyable":
 // it is ready to be applied and is represented by a diff.
-type NodeRefreshableResourceInstance struct {
+type NodeRefreshableManagedResourceInstance struct {
 	*NodeAbstractResource
 }
 
 // GraphNodeDestroyer
-func (n *NodeRefreshableResourceInstance) DestroyAddr() *ResourceAddress {
+func (n *NodeRefreshableManagedResourceInstance) DestroyAddr() *ResourceAddress {
 	return n.Addr
 }
 
 // GraphNodeEvalable
-func (n *NodeRefreshableResourceInstance) EvalTree() EvalNode {
+func (n *NodeRefreshableManagedResourceInstance) EvalTree() EvalNode {
 	// Eval info is different depending on what kind of resource this is
 	switch mode := n.Addr.Mode; mode {
 	case config.ManagedResourceMode:
@@ -44,7 +122,7 @@ func (n *NodeRefreshableResourceInstance) EvalTree() EvalNode {
 	}
 }
 
-func (n *NodeRefreshableResourceInstance) evalTreeManagedResource() EvalNode {
+func (n *NodeRefreshableManagedResourceInstance) evalTreeManagedResource() EvalNode {
 	addr := n.NodeAbstractResource.Addr
 
 	// stateId is the ID to put into the state

terraform/transform_resource_refresh_plannable.go

@@ -0,0 +1,55 @@
+package terraform
+
+import (
+	"fmt"
+	"log"
+)
+
+// ResourceRefreshPlannableTransformer is a GraphTransformer that replaces any
+// nodes that don't have state yet exist in config with
+// NodePlannableResourceInstance.
+//
+// This transformer is used when expanding count on managed resource nodes
+// during the refresh phase to ensure that data sources that have
+// interpolations that depend on resources existing in the graph can be walked
+// properly.
+type ResourceRefreshPlannableTransformer struct {
+	// The full global state.
+	State *State
+}
+
+// Transform implements GraphTransformer for
+// ResourceRefreshPlannableTransformer.
+func (t *ResourceRefreshPlannableTransformer) Transform(g *Graph) error {
+nextVertex:
+	for _, v := range g.Vertices() {
+		addr := v.(*NodeRefreshableManagedResourceInstance).Addr
+
+		// Find the state for this address, if there is one
+		filter := &StateFilter{State: t.State}
+		results, err := filter.Filter(addr.String())
+		if err != nil {
+			return err
+		}
+
+		// Check to see if we have a state for this resource. If we do, skip this
+		// node.
+		for _, result := range results {
+			if _, ok := result.Value.(*ResourceState); ok {
+				continue nextVertex
+			}
+		}
+		// If we don't, convert this resource to a NodePlannableResourceInstance node
+		// with all of the data we need to make it happen.
+		log.Printf("[TRACE] No state for %s, converting to NodePlannableResourceInstance", addr.String())
+		new := &NodePlannableResourceInstance{
+			NodeAbstractResource: v.(*NodeRefreshableManagedResourceInstance).NodeAbstractResource,
+		}
+		// Replace the node in the graph
+		if !g.Replace(v, new) {
+			return fmt.Errorf("ResourceRefreshPlannableTransformer: Could not replace node %#v with %#v", v, new)
+		}
+	}
+
+	return nil
+}