Merge pull request #9334 from hashicorp/f-shadow-graph

terraform: Shadow Graph

config/raw_config.go

@@ -62,6 +62,10 @@ func (r *RawConfig) RawMap() map[string]interface{} {
 
 // Copy returns a copy of this RawConfig, uninterpolated.
 func (r *RawConfig) Copy() *RawConfig {
+	if r == nil {
+		return nil
+	}
+
 	r.lock.Lock()
 	defer r.lock.Unlock()
 

helper/shadow/compared_value.go

@@ -0,0 +1,128 @@
+package shadow
+
+import (
+	"sync"
+)
+
+// ComparedValue is a struct that finds a value by comparing some key
+// to the list of stored values. This is useful when there is no easy
+// uniquely identifying key that works in a map (for that, use KeyedValue).
+//
+// ComparedValue is very expensive, relative to other Value types. Try to
+// limit the number of values stored in a ComparedValue by potentially
+// nesting it within a KeyedValue (a keyed value points to a compared value,
+// for example).
+type ComparedValue struct {
+	// Func is a function that is given the lookup key and a single
+	// stored value. If it matches, it returns true.
+	Func func(k, v interface{}) bool
+
+	lock    sync.Mutex
+	once    sync.Once
+	closed  bool
+	values  []interface{}
+	waiters map[interface{}]*Value
+}
+
+// Close closes the value. This can never fail. For a definition of
+// "close" see the ErrClosed docs.
+func (w *ComparedValue) Close() error {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// Set closed to true always
+	w.closed = true
+
+	// For all waiters, complete with ErrClosed
+	for k, val := range w.waiters {
+		val.SetValue(ErrClosed)
+		delete(w.waiters, k)
+	}
+
+	return nil
+}
+
+// Value returns the value that was set for the given key, or blocks
+// until one is available.
+func (w *ComparedValue) Value(k interface{}) interface{} {
+	v, val := w.valueWaiter(k)
+	if val == nil {
+		return v
+	}
+
+	return val.Value()
+}
+
+// ValueOk gets the value for the given key, returning immediately if the
+// value doesn't exist. The second return argument is true if the value exists.
+func (w *ComparedValue) ValueOk(k interface{}) (interface{}, bool) {
+	v, val := w.valueWaiter(k)
+	return v, val == nil
+}
+
+func (w *ComparedValue) SetValue(v interface{}) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	w.once.Do(w.init)
+
+	// Check if we already have this exact value (by simply comparing
+	// with == directly). If we do, then we don't insert it again.
+	found := false
+	for _, v2 := range w.values {
+		if v == v2 {
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		// Set the value, always
+		w.values = append(w.values, v)
+	}
+
+	// Go through the waiters
+	for k, val := range w.waiters {
+		if w.Func(k, v) {
+			val.SetValue(v)
+			delete(w.waiters, k)
+		}
+	}
+}
+
+func (w *ComparedValue) valueWaiter(k interface{}) (interface{}, *Value) {
+	w.lock.Lock()
+	w.once.Do(w.init)
+
+	// Look for a pre-existing value
+	for _, v := range w.values {
+		if w.Func(k, v) {
+			w.lock.Unlock()
+			return v, nil
+		}
+	}
+
+	// If we're closed, return that
+	if w.closed {
+		w.lock.Unlock()
+		return ErrClosed, nil
+	}
+
+	// Pre-existing value doesn't exist, create a waiter
+	val := w.waiters[k]
+	if val == nil {
+		val = new(Value)
+		w.waiters[k] = val
+	}
+	w.lock.Unlock()
+
+	// Return the waiter
+	return nil, val
+}
+
+// Must be called with w.lock held.
+func (w *ComparedValue) init() {
+	w.waiters = make(map[interface{}]*Value)
+	if w.Func == nil {
+		w.Func = func(k, v interface{}) bool { return k == v }
+	}
+}

helper/shadow/compared_value_test.go

@@ -0,0 +1,178 @@
+package shadow
+
+import (
+	"testing"
+	"time"
+)
+
+func TestComparedValue(t *testing.T) {
+	v := &ComparedValue{
+		Func: func(k, v interface{}) bool { return k == v },
+	}
+
+	// Start trying to get the value
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set the value
+	v.SetValue("foo")
+	val := <-valueCh
+
+	// Verify
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should get the next value
+	val = v.Value("foo")
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValue_setFirst(t *testing.T) {
+	v := &ComparedValue{
+		Func: func(k, v interface{}) bool { return k == v },
+	}
+
+	// Set the value
+	v.SetValue("foo")
+	val := v.Value("foo")
+
+	// Verify
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValueOk(t *testing.T) {
+	v := &ComparedValue{
+		Func: func(k, v interface{}) bool { return k == v },
+	}
+
+	// Try
+	val, ok := v.ValueOk("foo")
+	if ok {
+		t.Fatal("should not be ok")
+	}
+
+	// Set
+	v.SetValue("foo")
+
+	// Try again
+	val, ok = v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValueClose(t *testing.T) {
+	v := &ComparedValue{
+		Func: func(k, v interface{}) bool { return k == v },
+	}
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValueClose_blocked(t *testing.T) {
+	var v ComparedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Close
+	v.Close()
+
+	// Verify
+	val := <-valueCh
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValueClose_existing(t *testing.T) {
+	var v ComparedValue
+
+	// Set a value
+	v.SetValue("foo")
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestComparedValueClose_existingBlocked(t *testing.T) {
+	var v ComparedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// Wait
+	time.Sleep(10 * time.Millisecond)
+
+	// Set a value
+	v.SetValue("foo")
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != "foo" {
+		t.Fatalf("bad: %#v", val)
+	}
+}

helper/shadow/keyed_value.go

@@ -0,0 +1,150 @@
+package shadow
+
+import (
+	"sync"
+)
+
+// KeyedValue is a struct that coordinates a value by key. If a value is
+// not available for a give key, it'll block until it is available.
+type KeyedValue struct {
+	lock    sync.Mutex
+	once    sync.Once
+	values  map[string]interface{}
+	waiters map[string]*Value
+	closed  bool
+}
+
+// Close closes the value. This can never fail. For a definition of
+// "close" see the ErrClosed docs.
+func (w *KeyedValue) Close() error {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// Set closed to true always
+	w.closed = true
+
+	// For all waiters, complete with ErrClosed
+	for k, val := range w.waiters {
+		val.SetValue(ErrClosed)
+		delete(w.waiters, k)
+	}
+
+	return nil
+}
+
+// Value returns the value that was set for the given key, or blocks
+// until one is available.
+func (w *KeyedValue) Value(k string) interface{} {
+	w.lock.Lock()
+	v, val := w.valueWaiter(k)
+	w.lock.Unlock()
+
+	// If we have no waiter, then return the value
+	if val == nil {
+		return v
+	}
+
+	// We have a waiter, so wait
+	return val.Value()
+}
+
+// WaitForChange waits for the value with the given key to be set again.
+// If the key isn't set, it'll wait for an initial value. Note that while
+// it is called "WaitForChange", the value isn't guaranteed to _change_;
+// this will return when a SetValue is called for the given k.
+func (w *KeyedValue) WaitForChange(k string) interface{} {
+	w.lock.Lock()
+	w.once.Do(w.init)
+
+	// If we're closed, we're closed
+	if w.closed {
+		return ErrClosed
+	}
+
+	// Check for an active waiter. If there isn't one, make it
+	val := w.waiters[k]
+	if val == nil {
+		val = new(Value)
+		w.waiters[k] = val
+	}
+	w.lock.Unlock()
+
+	// And wait
+	return val.Value()
+}
+
+// ValueOk gets the value for the given key, returning immediately if the
+// value doesn't exist. The second return argument is true if the value exists.
+func (w *KeyedValue) ValueOk(k string) (interface{}, bool) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	v, val := w.valueWaiter(k)
+	return v, val == nil
+}
+
+func (w *KeyedValue) SetValue(k string, v interface{}) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+	w.setValue(k, v)
+}
+
+// Init will initialize the key to a given value only if the key has
+// not been set before. This is safe to call multiple times and in parallel.
+func (w *KeyedValue) Init(k string, v interface{}) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// If we have a waiter, set the value.
+	_, val := w.valueWaiter(k)
+	if val != nil {
+		w.setValue(k, v)
+	}
+}
+
+// Must be called with w.lock held.
+func (w *KeyedValue) init() {
+	w.values = make(map[string]interface{})
+	w.waiters = make(map[string]*Value)
+}
+
+// setValue is like SetValue but assumes the lock is held.
+func (w *KeyedValue) setValue(k string, v interface{}) {
+	w.once.Do(w.init)
+
+	// Set the value, always
+	w.values[k] = v
+
+	// If we have a waiter, set it
+	if val, ok := w.waiters[k]; ok {
+		val.SetValue(v)
+		delete(w.waiters, k)
+	}
+}
+
+// valueWaiter gets the value or the Value waiter for a given key.
+//
+// This must be called with lock held.
+func (w *KeyedValue) valueWaiter(k string) (interface{}, *Value) {
+	w.once.Do(w.init)
+
+	// If we have this value already, return it
+	if v, ok := w.values[k]; ok {
+		return v, nil
+	}
+
+	// If we're closed, return that
+	if w.closed {
+		return ErrClosed, nil
+	}
+
+	// No pending value, check for a waiter
+	val := w.waiters[k]
+	if val == nil {
+		val = new(Value)
+		w.waiters[k] = val
+	}
+
+	// Return the waiter
+	return nil, val
+}

helper/shadow/keyed_value_test.go

@@ -0,0 +1,314 @@
+package shadow
+
+import (
+	"testing"
+	"time"
+)
+
+func TestKeyedValue(t *testing.T) {
+	var v KeyedValue
+
+	// Start trying to get the value
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set the value
+	v.SetValue("foo", 42)
+	val := <-valueCh
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should get the next value
+	val = v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValue_setFirst(t *testing.T) {
+	var v KeyedValue
+
+	// Set the value
+	v.SetValue("foo", 42)
+	val := v.Value("foo")
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueOk(t *testing.T) {
+	var v KeyedValue
+
+	// Try
+	val, ok := v.ValueOk("foo")
+	if ok {
+		t.Fatal("should not be ok")
+	}
+
+	// Set
+	v.SetValue("foo", 42)
+
+	// Try again
+	val, ok = v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueClose(t *testing.T) {
+	var v KeyedValue
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueClose_blocked(t *testing.T) {
+	var v KeyedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Close
+	v.Close()
+
+	// Verify
+	val := <-valueCh
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueClose_existing(t *testing.T) {
+	var v KeyedValue
+
+	// Set a value
+	v.SetValue("foo", "bar")
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != "bar" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueClose_existingBlocked(t *testing.T) {
+	var v KeyedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value("foo")
+	}()
+
+	// Wait
+	time.Sleep(10 * time.Millisecond)
+
+	// Set a value
+	v.SetValue("foo", "bar")
+
+	// Close
+	v.Close()
+
+	// Try again
+	val, ok := v.ValueOk("foo")
+	if !ok {
+		t.Fatal("should be ok")
+	}
+
+	// Verify
+	if val != "bar" {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueInit(t *testing.T) {
+	var v KeyedValue
+
+	v.Init("foo", 42)
+
+	// We should get the value
+	val := v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should get the value
+	val = v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// This should do nothing
+	v.Init("foo", 84)
+
+	// We should get the value
+	val = v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueInit_set(t *testing.T) {
+	var v KeyedValue
+
+	v.SetValue("foo", 42)
+
+	// We should get the value
+	val := v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should get the value
+	val = v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// This should do nothing
+	v.Init("foo", 84)
+
+	// We should get the value
+	val = v.Value("foo")
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueWaitForChange(t *testing.T) {
+	var v KeyedValue
+
+	// Set a value
+	v.SetValue("foo", 42)
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.WaitForChange("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set a new value
+	v.SetValue("foo", 84)
+
+	// Verify
+	val := <-valueCh
+	if val != 84 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueWaitForChange_initial(t *testing.T) {
+	var v KeyedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.WaitForChange("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set a new value
+	v.SetValue("foo", 84)
+
+	// Verify
+	val := <-valueCh
+	if val != 84 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestKeyedValueWaitForChange_closed(t *testing.T) {
+	var v KeyedValue
+
+	// Start reading this should be blocking
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.WaitForChange("foo")
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Close
+	v.Close()
+
+	// Verify
+	val := <-valueCh
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// Set a value
+	v.SetValue("foo", 42)
+
+	// Try again
+	val = v.WaitForChange("foo")
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}

helper/shadow/ordered_value.go

@@ -0,0 +1,66 @@
+package shadow
+
+import (
+	"container/list"
+	"sync"
+)
+
+// OrderedValue is a struct that keeps track of a value in the order
+// it is set. Each time Value() is called, it will return the most recent
+// calls value then discard it.
+//
+// This is unlike Value that returns the same value once it is set.
+type OrderedValue struct {
+	lock    sync.Mutex
+	values  *list.List
+	waiters *list.List
+}
+
+// Value returns the last value that was set, or blocks until one
+// is received.
+func (w *OrderedValue) Value() interface{} {
+	w.lock.Lock()
+
+	// If we have a pending value already, use it
+	if w.values != nil && w.values.Len() > 0 {
+		front := w.values.Front()
+		w.values.Remove(front)
+		w.lock.Unlock()
+		return front.Value
+	}
+
+	// No pending value, create a waiter
+	if w.waiters == nil {
+		w.waiters = list.New()
+	}
+
+	var val Value
+	w.waiters.PushBack(&val)
+	w.lock.Unlock()
+
+	// Return the value once we have it
+	return val.Value()
+}
+
+// SetValue sets the latest value.
+func (w *OrderedValue) SetValue(v interface{}) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// If we have a waiter, notify it
+	if w.waiters != nil && w.waiters.Len() > 0 {
+		front := w.waiters.Front()
+		w.waiters.Remove(front)
+
+		val := front.Value.(*Value)
+		val.SetValue(v)
+		return
+	}
+
+	// Add it to the list of values
+	if w.values == nil {
+		w.values = list.New()
+	}
+
+	w.values.PushBack(v)
+}

helper/shadow/ordered_value_test.go

@@ -0,0 +1,76 @@
+package shadow
+
+import (
+	"testing"
+	"time"
+)
+
+func TestOrderedValue(t *testing.T) {
+	var v OrderedValue
+
+	// Start trying to get the value
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value()
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set the value
+	v.SetValue(42)
+	val := <-valueCh
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should not get the value again
+	go func() {
+		valueCh <- v.Value()
+	}()
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// We should get the next value
+	v.SetValue(21)
+	val = <-valueCh
+	if val != 21 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestOrderedValue_setFirst(t *testing.T) {
+	var v OrderedValue
+
+	// Set the value
+	v.SetValue(42)
+	val := v.Value()
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should not get the value again
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value()
+	}()
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set a value so the goroutine doesn't hang around
+	v.SetValue(1)
+}

helper/shadow/value.go

@@ -0,0 +1,79 @@
+package shadow
+
+import (
+	"errors"
+	"sync"
+)
+
+// ErrClosed is returned by any closed values.
+//
+// A "closed value" is when the shadow has been notified that the real
+// side is complete and any blocking values will _never_ be satisfied
+// in the future. In this case, this error is returned. If a value is already
+// available, that is still returned.
+var ErrClosed = errors.New("shadow closed")
+
+// Value is a struct that coordinates a value between two
+// parallel routines. It is similar to atomic.Value except that when
+// Value is called if it isn't set it will wait for it.
+//
+// The Value can be closed with Close, which will cause any future
+// blocking operations to return immediately with ErrClosed.
+type Value struct {
+	lock     sync.Mutex
+	cond     *sync.Cond
+	value    interface{}
+	valueSet bool
+}
+
+// Close closes the value. This can never fail. For a definition of
+// "close" see the struct docs.
+func (w *Value) Close() error {
+	w.lock.Lock()
+	set := w.valueSet
+	w.lock.Unlock()
+
+	// If we haven't set the value, set it
+	if !set {
+		w.SetValue(ErrClosed)
+	}
+
+	// Done
+	return nil
+}
+
+// Value returns the value that was set.
+func (w *Value) Value() interface{} {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// If we already have a value just return
+	for !w.valueSet {
+		// No value, setup the condition variable if we have to
+		if w.cond == nil {
+			w.cond = sync.NewCond(&w.lock)
+		}
+
+		// Wait on it
+		w.cond.Wait()
+	}
+
+	// Return the value
+	return w.value
+}
+
+// SetValue sets the value.
+func (w *Value) SetValue(v interface{}) {
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	// Set the value
+	w.valueSet = true
+	w.value = v
+
+	// If we have a condition, clear it
+	if w.cond != nil {
+		w.cond.Broadcast()
+		w.cond = nil
+	}
+}

helper/shadow/value_test.go

@@ -0,0 +1,103 @@
+package shadow
+
+import (
+	"testing"
+	"time"
+)
+
+func TestValue(t *testing.T) {
+	var v Value
+
+	// Start trying to get the value
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value()
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set the value
+	v.SetValue(42)
+	val := <-valueCh
+
+	// Verify
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should be able to ask for the value again immediately
+	if val := v.Value(); val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We can change the value
+	v.SetValue(84)
+	if val := v.Value(); val != 84 {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestValueClose(t *testing.T) {
+	var v Value
+
+	// Close
+	v.Close()
+
+	// Verify
+	val := v.Value()
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestValueClose_blocked(t *testing.T) {
+	var v Value
+
+	// Start trying to get the value
+	valueCh := make(chan interface{})
+	go func() {
+		valueCh <- v.Value()
+	}()
+
+	// We should not get the value
+	select {
+	case <-valueCh:
+		t.Fatal("shouldn't receive value")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Set the value
+	v.Close()
+	val := <-valueCh
+
+	// Verify
+	if val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+
+	// We should be able to ask for the value again immediately
+	if val := v.Value(); val != ErrClosed {
+		t.Fatalf("bad: %#v", val)
+	}
+}
+
+func TestValueClose_existing(t *testing.T) {
+	var v Value
+
+	// Set the value
+	v.SetValue(42)
+
+	// Close
+	v.Close()
+
+	// Verify
+	val := v.Value()
+	if val != 42 {
+		t.Fatalf("bad: %#v", val)
+	}
+}

terraform/context.go

@@ -32,6 +32,17 @@ const (
 	InputModeStd = InputModeVar | InputModeProvider
 )
 
+var (
+	// contextFailOnShadowError will cause Context operations to return
+	// errors when shadow operations fail. This is only used for testing.
+	contextFailOnShadowError = false
+
+	// contextTestDeepCopyOnPlan will perform a Diff DeepCopy on every
+	// Plan operation, effectively testing the Diff DeepCopy whenever
+	// a Plan occurs. This is enabled for tests.
+	contextTestDeepCopyOnPlan = false
+)
+
 // ContextOpts are the user-configurable options to create a context with
 // NewContext.
 type ContextOpts struct {
@@ -56,24 +67,28 @@ type ContextOpts struct {
 //
 // Extra functions on Context can be found in context_*.go files.
 type Context struct {
-	destroy      bool
+	// Maintainer note: Anytime this struct is changed, please verify
-	diff         *Diff
+	// that newShadowContext still does the right thing. Tests should
-	diffLock     sync.RWMutex
+	// fail regardless but putting this note here as well.
-	hooks        []Hook
+
-	module       *module.Tree
+	components contextComponentFactory
-	providers    map[string]ResourceProviderFactory
+	destroy    bool
-	provisioners map[string]ResourceProvisionerFactory
+	diff       *Diff
-	sh           *stopHook
+	diffLock   sync.RWMutex
-	state        *State
+	hooks      []Hook
-	stateLock    sync.RWMutex
+	module     *module.Tree
-	targets      []string
+	sh         *stopHook
-	uiInput      UIInput
+	state      *State
-	variables    map[string]interface{}
+	stateLock  sync.RWMutex
+	targets    []string
+	uiInput    UIInput
+	variables  map[string]interface{}
 
 	l                   sync.Mutex // Lock acquired during any task
 	parallelSem         Semaphore
 	providerInputConfig map[string]map[string]interface{}
 	runCh               <-chan struct{}
+	shadowErr           error
 }
 
 // NewContext creates a new Context structure.
@@ -136,16 +151,18 @@ func NewContext(opts *ContextOpts) (*Context, error) {
 	}
 
 	return &Context{
-		destroy:      opts.Destroy,
+		components: &basicComponentFactory{
-		diff:         opts.Diff,
+			providers:    opts.Providers,
-		hooks:        hooks,
+			provisioners: opts.Provisioners,
-		module:       opts.Module,
+		},
-		providers:    opts.Providers,
+		destroy:   opts.Destroy,
-		provisioners: opts.Provisioners,
+		diff:      opts.Diff,
-		state:        state,
+		hooks:     hooks,
-		targets:      opts.Targets,
+		module:    opts.Module,
-		uiInput:      opts.UIInput,
+		state:     state,
-		variables:    variables,
+		targets:   opts.Targets,
+		uiInput:   opts.UIInput,
+		variables: variables,
 
 		parallelSem:         NewSemaphore(par),
 		providerInputConfig: make(map[string]map[string]interface{}),
@@ -166,22 +183,11 @@ func (c *Context) Graph(g *ContextGraphOpts) (*Graph, error) {
 // GraphBuilder returns the GraphBuilder that will be used to create
 // the graphs for this context.
 func (c *Context) graphBuilder(g *ContextGraphOpts) GraphBuilder {
-	// TODO test
-	providers := make([]string, 0, len(c.providers))
-	for k, _ := range c.providers {
-		providers = append(providers, k)
-	}
-
-	provisioners := make([]string, 0, len(c.provisioners))
-	for k, _ := range c.provisioners {
-		provisioners = append(provisioners, k)
-	}
-
 	return &BuiltinGraphBuilder{
 		Root:         c.module,
 		Diff:         c.diff,
-		Providers:    providers,
+		Providers:    c.components.ResourceProviders(),
-		Provisioners: provisioners,
+		Provisioners: c.components.ResourceProvisioners(),
 		State:        c.state,
 		Targets:      c.targets,
 		Destroy:      c.destroy,
@@ -190,6 +196,33 @@ func (c *Context) graphBuilder(g *ContextGraphOpts) GraphBuilder {
 	}
 }
 
+// ShadowError returns any errors caught during a shadow operation.
+//
+// A shadow operation is an operation run in parallel to a real operation
+// that performs the same tasks using new logic on copied state. The results
+// are compared to ensure that the new logic works the same as the old logic.
+// The shadow never affects the real operation or return values.
+//
+// The result of the shadow operation are only available through this function
+// call after a real operation is complete.
+//
+// For API consumers of Context, you can safely ignore this function
+// completely if you have no interest in helping report experimental feature
+// errors to Terraform maintainers. Otherwise, please call this function
+// after every operation and report this to the user.
+//
+// IMPORTANT: Shadow errors are _never_ critical: they _never_ affect
+// the real state or result of a real operation. They are purely informational
+// to assist in future Terraform versions being more stable. Please message
+// this effectively to the end user.
+//
+// This must be called only when no other operation is running (refresh,
+// plan, etc.). The result can be used in parallel to any other operation
+// running.
+func (c *Context) ShadowError() error {
+	return c.shadowErr
+}
+
 // Input asks for input to fill variables and provider configurations.
 // This modifies the configuration in-place, so asking for Input twice
 // may result in different UI output showing different current values.
@@ -300,7 +333,7 @@ func (c *Context) Input(mode InputMode) error {
 		}
 
 		// Do the walk
-		if _, err := c.walk(graph, walkInput); err != nil {
+		if _, err := c.walk(graph, nil, walkInput); err != nil {
 			return err
 		}
 	}
@@ -329,9 +362,10 @@ func (c *Context) Apply() (*State, error) {
 	// Do the walk
 	var walker *ContextGraphWalker
 	if c.destroy {
-		walker, err = c.walk(graph, walkDestroy)
+		walker, err = c.walk(graph, graph, walkDestroy)
 	} else {
-		walker, err = c.walk(graph, walkApply)
+		//walker, err = c.walk(graph, nil, walkApply)
+		walker, err = c.walk(graph, graph, walkApply)
 	}
 
 	if len(walker.ValidationErrors) > 0 {
@@ -396,12 +430,23 @@ func (c *Context) Plan() (*Plan, error) {
 	}
 
 	// Do the walk
-	walker, err := c.walk(graph, operation)
+	walker, err := c.walk(graph, graph, operation)
 	if err != nil {
 		return nil, err
 	}
 	p.Diff = c.diff
 
+	// If this is true, it means we're running unit tests. In this case,
+	// we perform a deep copy just to ensure that all context tests also
+	// test that a diff is copy-able. This will panic if it fails. This
+	// is enabled during unit tests.
+	//
+	// This should never be true during production usage, but even if it is,
+	// it can't do any real harm.
+	if contextTestDeepCopyOnPlan {
+		p.Diff.DeepCopy()
+	}
+
 	// Now that we have a diff, we can build the exact graph that Apply will use
 	// and catch any possible cycles during the Plan phase.
 	if _, err := c.Graph(&ContextGraphOpts{Validate: true}); err != nil {
@@ -434,7 +479,7 @@ func (c *Context) Refresh() (*State, error) {
 	}
 
 	// Do the walk
-	if _, err := c.walk(graph, walkRefresh); err != nil {
+	if _, err := c.walk(graph, graph, walkRefresh); err != nil {
 		return nil, err
 	}
 
@@ -502,7 +547,7 @@ func (c *Context) Validate() ([]string, []error) {
 	}
 
 	// Walk
-	walker, err := c.walk(graph, walkValidate)
+	walker, err := c.walk(graph, graph, walkValidate)
 	if err != nil {
 		return nil, multierror.Append(errs, err).Errors
 	}
@@ -541,8 +586,16 @@ func (c *Context) acquireRun() chan<- struct{} {
 		c.l.Lock()
 	}
 
+	// Create the new channel
 	ch := make(chan struct{})
 	c.runCh = ch
+
+	// Reset the stop hook so we're not stopped
+	c.sh.Reset()
+
+	// Reset the shadow errors
+	c.shadowErr = nil
+
 	return ch
 }
 
@@ -552,15 +605,108 @@ func (c *Context) releaseRun(ch chan<- struct{}) {
 
 	close(ch)
 	c.runCh = nil
-	c.sh.Reset()
 }
 
 func (c *Context) walk(
-	graph *Graph, operation walkOperation) (*ContextGraphWalker, error) {
+	graph, shadow *Graph, operation walkOperation) (*ContextGraphWalker, error) {
-	// Walk the graph
+	// Keep track of the "real" context which is the context that does
+	// the real work: talking to real providers, modifying real state, etc.
+	realCtx := c
+
+	// If we have a shadow graph, walk that as well
+	var shadowCtx *Context
+	var shadowCloser Shadow
+	if shadow != nil {
+		// Build the shadow context. In the process, override the real context
+		// with the one that is wrapped so that the shadow context can verify
+		// the results of the real.
+		realCtx, shadowCtx, shadowCloser = newShadowContext(c)
+	}
+
+	// Build the real graph walker
 	log.Printf("[DEBUG] Starting graph walk: %s", operation.String())
-	walker := &ContextGraphWalker{Context: c, Operation: operation}
+	walker := &ContextGraphWalker{Context: realCtx, Operation: operation}
-	return walker, graph.Walk(walker)
+
+	// Walk the real graph, this will block until it completes
+	realErr := graph.Walk(walker)
+
+	// If we have a shadow graph and we interrupted the real graph, then
+	// we just close the shadow and never verify it. It is non-trivial to
+	// recreate the exact execution state up until an interruption so this
+	// isn't supported with shadows at the moment.
+	if shadowCloser != nil && c.sh.Stopped() {
+		// Ignore the error result, there is nothing we could care about
+		shadowCloser.CloseShadow()
+
+		// Set it to nil so we don't do anything
+		shadowCloser = nil
+	}
+
+	// If we have a shadow graph, wait for that to complete.
+	if shadowCloser != nil {
+		// Build the graph walker for the shadow.
+		shadowWalker := &ContextGraphWalker{
+			Context:   shadowCtx,
+			Operation: operation,
+		}
+
+		// Kick off the shadow walk. This will block on any operations
+		// on the real walk so it is fine to start first.
+		shadowCh := make(chan error)
+		go func() {
+			log.Printf("[INFO] Starting shadow graph walk: %s", operation.String())
+			shadowCh <- shadow.Walk(shadowWalker)
+		}()
+
+		// Notify the shadow that we're done
+		if err := shadowCloser.CloseShadow(); err != nil {
+			c.shadowErr = multierror.Append(c.shadowErr, err)
+		}
+
+		// Wait for the walk to end
+		log.Printf("[DEBUG] Waiting for shadow graph to complete...")
+		shadowWalkErr := <-shadowCh
+
+		// Get any shadow errors
+		if err := shadowCloser.ShadowError(); err != nil {
+			c.shadowErr = multierror.Append(c.shadowErr, err)
+		}
+
+		// Verify the contexts (compare)
+		if err := shadowContextVerify(realCtx, shadowCtx); err != nil {
+			c.shadowErr = multierror.Append(c.shadowErr, err)
+		}
+
+		// At this point, if we're supposed to fail on error, then
+		// we PANIC. Some tests just verify that there is an error,
+		// so simply appending it to realErr and returning could hide
+		// shadow problems.
+		//
+		// This must be done BEFORE appending shadowWalkErr since the
+		// shadowWalkErr may include expected errors.
+		if c.shadowErr != nil && contextFailOnShadowError {
+			panic(multierror.Prefix(c.shadowErr, "shadow graph:"))
+		}
+
+		// Now, if we have a walk error, we append that through
+		if shadowWalkErr != nil {
+			c.shadowErr = multierror.Append(c.shadowErr, shadowWalkErr)
+		}
+
+		if c.shadowErr == nil {
+			log.Printf("[INFO] Shadow graph success!")
+		} else {
+			log.Printf("[ERROR] Shadow graph error: %s", c.shadowErr)
+
+			// If we're supposed to fail on shadow errors, then report it
+			if contextFailOnShadowError {
+				realErr = multierror.Append(realErr, multierror.Prefix(
+					c.shadowErr, "shadow graph:"))
+			}
+		}
+	}
+
+	return walker, realErr
 }
 
 // parseVariableAsHCL parses the value of a single variable as would have been specified

terraform/context_apply_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/terraform/config/module"
 )
 
-func TestContext2Apply(t *testing.T) {
+func TestContext2Apply_basic(t *testing.T) {
 	m := testModule(t, "apply-good")
 	p := testProvider("aws")
 	p.ApplyFn = testApplyFn
@@ -318,10 +318,10 @@ func TestContext2Apply_computedAttrRefTypeMismatch(t *testing.T) {
 	}
 
 	_, err := ctx.Apply()
-
 	if err == nil {
 		t.Fatalf("Expected err, got none!")
 	}
+
 	expected := "Expected ami to be string"
 	if !strings.Contains(err.Error(), expected) {
 		t.Fatalf("expected:\n\n%s\n\nto contain:\n\n%s", err, expected)

terraform/context_components.go

@@ -0,0 +1,65 @@
+package terraform
+
+import (
+	"fmt"
+)
+
+// contextComponentFactory is the interface that Context uses
+// to initialize various components such as providers and provisioners.
+// This factory gets more information than the raw maps using to initialize
+// a Context. This information is used for debugging.
+type contextComponentFactory interface {
+	// ResourceProvider creates a new ResourceProvider with the given
+	// type. The "uid" is a unique identifier for this provider being
+	// initialized that can be used for internal tracking.
+	ResourceProvider(typ, uid string) (ResourceProvider, error)
+	ResourceProviders() []string
+
+	// ResourceProvisioner creates a new ResourceProvisioner with the
+	// given type. The "uid" is a unique identifier for this provisioner
+	// being initialized that can be used for internal tracking.
+	ResourceProvisioner(typ, uid string) (ResourceProvisioner, error)
+	ResourceProvisioners() []string
+}
+
+// basicComponentFactory just calls a factory from a map directly.
+type basicComponentFactory struct {
+	providers    map[string]ResourceProviderFactory
+	provisioners map[string]ResourceProvisionerFactory
+}
+
+func (c *basicComponentFactory) ResourceProviders() []string {
+	result := make([]string, len(c.providers))
+	for k, _ := range c.providers {
+		result = append(result, k)
+	}
+
+	return result
+}
+
+func (c *basicComponentFactory) ResourceProvisioners() []string {
+	result := make([]string, len(c.provisioners))
+	for k, _ := range c.provisioners {
+		result = append(result, k)
+	}
+
+	return result
+}
+
+func (c *basicComponentFactory) ResourceProvider(typ, uid string) (ResourceProvider, error) {
+	f, ok := c.providers[typ]
+	if !ok {
+		return nil, fmt.Errorf("unknown provider %q", typ)
+	}
+
+	return f()
+}
+
+func (c *basicComponentFactory) ResourceProvisioner(typ, uid string) (ResourceProvisioner, error) {
+	f, ok := c.provisioners[typ]
+	if !ok {
+		return nil, fmt.Errorf("unknown provisioner %q", typ)
+	}
+
+	return f()
+}

terraform/context_import.go

@@ -43,17 +43,11 @@ func (c *Context) Import(opts *ImportOpts) (*State, error) {
 	// Copy our own state
 	c.state = c.state.DeepCopy()
 
-	// Get supported providers (for the graph builder)
-	providers := make([]string, 0, len(c.providers))
-	for k, _ := range c.providers {
-		providers = append(providers, k)
-	}
-
 	// Initialize our graph builder
 	builder := &ImportGraphBuilder{
 		ImportTargets: opts.Targets,
 		Module:        opts.Module,
-		Providers:     providers,
+		Providers:     c.components.ResourceProviders(),
 	}
 
 	// Build the graph!
@@ -63,7 +57,7 @@ func (c *Context) Import(opts *ImportOpts) (*State, error) {
 	}
 
 	// Walk it
-	if _, err := c.walk(graph, walkImport); err != nil {
+	if _, err := c.walk(graph, nil, walkImport); err != nil {
 		return c.state, err
 	}
 

terraform/context_plan_test.go

@@ -1088,6 +1088,10 @@ func TestContext2Plan_dataResourceBecomesComputed(t *testing.T) {
 		t.Fatalf("missing diff for data.aws_data_resource.foo")
 	}
 
+	// This is added by the diff but we want to verify that we got
+	// the same diff as above minus the dynamic stuff.
+	delete(iDiff.Attributes, "id")
+
 	if same, _ := p.ReadDataDiffReturn.Same(iDiff); !same {
 		t.Fatalf(
 			"incorrect diff for data.data_resource.foo\ngot:  %#v\nwant: %#v",

terraform/diff.go

@@ -9,6 +9,8 @@ import (
 	"sort"
 	"strings"
 	"sync"
+
+	"github.com/mitchellh/copystructure"
 )
 
 // DiffChangeType is an enum with the kind of changes a diff has planned.
@@ -79,6 +81,36 @@ func (d *Diff) Empty() bool {
 	return true
 }
 
+// Equal compares two diffs for exact equality.
+//
+// This is different from the Same comparison that is supported which
+// checks for operation equality taking into account computed values. Equal
+// instead checks for exact equality.
+func (d *Diff) Equal(d2 *Diff) bool {
+	// If one is nil, they must both be nil
+	if d == nil || d2 == nil {
+		return d == d2
+	}
+
+	// Sort the modules
+	sort.Sort(moduleDiffSort(d.Modules))
+	sort.Sort(moduleDiffSort(d2.Modules))
+
+	// Use DeepEqual
+	return reflect.DeepEqual(d, d2)
+}
+
+// DeepCopy performs a deep copy of all parts of the Diff, making the
+// resulting Diff safe to use without modifying this one.
+func (d *Diff) DeepCopy() *Diff {
+	copy, err := copystructure.Config{Lock: true}.Copy(d)
+	if err != nil {
+		panic(err)
+	}
+
+	return copy.(*Diff)
+}
+
 func (d *Diff) String() string {
 	var buf bytes.Buffer
 
@@ -364,6 +396,31 @@ func (d *InstanceDiff) Empty() bool {
 	return !d.Destroy && !d.DestroyTainted && len(d.Attributes) == 0
 }
 
+// Equal compares two diffs for exact equality.
+//
+// This is different from the Same comparison that is supported which
+// checks for operation equality taking into account computed values. Equal
+// instead checks for exact equality.
+func (d *InstanceDiff) Equal(d2 *InstanceDiff) bool {
+	// If one is nil, they must both be nil
+	if d == nil || d2 == nil {
+		return d == d2
+	}
+
+	// Use DeepEqual
+	return reflect.DeepEqual(d, d2)
+}
+
+// DeepCopy performs a deep copy of all parts of the InstanceDiff
+func (d *InstanceDiff) DeepCopy() *InstanceDiff {
+	copy, err := copystructure.Config{Lock: true}.Copy(d)
+	if err != nil {
+		panic(err)
+	}
+
+	return copy.(*InstanceDiff)
+}
+
 func (d *InstanceDiff) GoString() string {
 	return fmt.Sprintf("*%#v", InstanceDiff{
 		Attributes:     d.Attributes,
@@ -632,3 +689,21 @@ func (d *InstanceDiff) Same(d2 *InstanceDiff) (bool, string) {
 
 	return true, ""
 }
+
+// moduleDiffSort implements sort.Interface to sort module diffs by path.
+type moduleDiffSort []*ModuleDiff
+
+func (s moduleDiffSort) Len() int      { return len(s) }
+func (s moduleDiffSort) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s moduleDiffSort) Less(i, j int) bool {
+	a := s[i]
+	b := s[j]
+
+	// If the lengths are different, then the shorter one always wins
+	if len(a.Path) != len(b.Path) {
+		return len(a.Path) < len(b.Path)
+	}
+
+	// Otherwise, compare lexically
+	return strings.Join(a.Path, ".") < strings.Join(b.Path, ".")
+}

terraform/diff_test.go

@@ -40,6 +40,50 @@ func TestDiffEmpty_taintedIsNotEmpty(t *testing.T) {
 	}
 }
 
+func TestDiffEqual(t *testing.T) {
+	cases := map[string]struct {
+		D1, D2 *Diff
+		Equal  bool
+	}{
+		"nil": {
+			nil,
+			new(Diff),
+			false,
+		},
+
+		"empty": {
+			new(Diff),
+			new(Diff),
+			true,
+		},
+
+		"different module order": {
+			&Diff{
+				Modules: []*ModuleDiff{
+					&ModuleDiff{Path: []string{"root", "foo"}},
+					&ModuleDiff{Path: []string{"root", "bar"}},
+				},
+			},
+			&Diff{
+				Modules: []*ModuleDiff{
+					&ModuleDiff{Path: []string{"root", "bar"}},
+					&ModuleDiff{Path: []string{"root", "foo"}},
+				},
+			},
+			true,
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			actual := tc.D1.Equal(tc.D2)
+			if actual != tc.Equal {
+				t.Fatalf("expected: %v\n\n%#v\n\n%#v", tc.Equal, tc.D1, tc.D2)
+			}
+		})
+	}
+}
+
 func TestModuleDiff_ChangeType(t *testing.T) {
 	cases := []struct {
 		Diff   *ModuleDiff
@@ -115,6 +159,39 @@ func TestModuleDiff_ChangeType(t *testing.T) {
 	}
 }
 
+func TestDiff_DeepCopy(t *testing.T) {
+	cases := map[string]*Diff{
+		"empty": &Diff{},
+
+		"basic diff": &Diff{
+			Modules: []*ModuleDiff{
+				&ModuleDiff{
+					Path: []string{"root"},
+					Resources: map[string]*InstanceDiff{
+						"aws_instance.foo": &InstanceDiff{
+							Attributes: map[string]*ResourceAttrDiff{
+								"num": &ResourceAttrDiff{
+									Old: "0",
+									New: "2",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dup := tc.DeepCopy()
+			if !reflect.DeepEqual(dup, tc) {
+				t.Fatalf("\n%#v\n\n%#v", dup, tc)
+			}
+		})
+	}
+}
+
 func TestModuleDiff_Empty(t *testing.T) {
 	diff := new(ModuleDiff)
 	if !diff.Empty() {

terraform/eval_context_builtin.go

@@ -26,14 +26,13 @@ type BuiltinEvalContext struct {
 	InterpolaterVars    map[string]map[string]interface{}
 	InterpolaterVarLock *sync.Mutex
 
+	Components          contextComponentFactory
 	Hooks               []Hook
 	InputValue          UIInput
-	Providers           map[string]ResourceProviderFactory
 	ProviderCache       map[string]ResourceProvider
 	ProviderConfigCache map[string]*ResourceConfig
 	ProviderInputConfig map[string]map[string]interface{}
 	ProviderLock        *sync.Mutex
-	Provisioners        map[string]ResourceProvisionerFactory
 	ProvisionerCache    map[string]ResourceProvisioner
 	ProvisionerLock     *sync.Mutex
 	DiffValue           *Diff
@@ -81,23 +80,18 @@ func (ctx *BuiltinEvalContext) InitProvider(n string) (ResourceProvider, error)
 	ctx.ProviderLock.Lock()
 	defer ctx.ProviderLock.Unlock()
 
+	providerPath := make([]string, len(ctx.Path())+1)
+	copy(providerPath, ctx.Path())
+	providerPath[len(providerPath)-1] = n
+	key := PathCacheKey(providerPath)
+
 	typeName := strings.SplitN(n, ".", 2)[0]
-
+	p, err := ctx.Components.ResourceProvider(typeName, key)
-	f, ok := ctx.Providers[typeName]
-	if !ok {
-		return nil, fmt.Errorf("Provider '%s' not found", typeName)
-	}
-
-	p, err := f()
 	if err != nil {
 		return nil, err
 	}
 
-	providerPath := make([]string, len(ctx.Path())+1)
+	ctx.ProviderCache[key] = p
-	copy(providerPath, ctx.Path())
-	providerPath[len(providerPath)-1] = n
-
-	ctx.ProviderCache[PathCacheKey(providerPath)] = p
 	return p, nil
 }
 
@@ -231,21 +225,17 @@ func (ctx *BuiltinEvalContext) InitProvisioner(
 	ctx.ProvisionerLock.Lock()
 	defer ctx.ProvisionerLock.Unlock()
 
-	f, ok := ctx.Provisioners[n]
+	provPath := make([]string, len(ctx.Path())+1)
-	if !ok {
+	copy(provPath, ctx.Path())
-		return nil, fmt.Errorf("Provisioner '%s' not found", n)
+	provPath[len(provPath)-1] = n
-	}
+	key := PathCacheKey(provPath)
 
-	p, err := f()
+	p, err := ctx.Components.ResourceProvisioner(n, key)
 	if err != nil {
 		return nil, err
 	}
 
-	provPath := make([]string, len(ctx.Path())+1)
+	ctx.ProvisionerCache[key] = p
-	copy(provPath, ctx.Path())
-	provPath[len(provPath)-1] = n
-
-	ctx.ProvisionerCache[PathCacheKey(provPath)] = p
 	return p, nil
 }
 
@@ -341,9 +331,4 @@ func (ctx *BuiltinEvalContext) State() (*State, *sync.RWMutex) {
 }
 
 func (ctx *BuiltinEvalContext) init() {
-	// We nil-check the things below because they're meant to be configured,
-	// and we just default them to non-nil.
-	if ctx.Providers == nil {
-		ctx.Providers = make(map[string]ResourceProviderFactory)
-	}
 }

terraform/graph_walk_context.go

@@ -68,12 +68,11 @@ func (w *ContextGraphWalker) EnterPath(path []string) EvalContext {
 		PathValue:           path,
 		Hooks:               w.Context.hooks,
 		InputValue:          w.Context.uiInput,
-		Providers:           w.Context.providers,
+		Components:          w.Context.components,
 		ProviderCache:       w.providerCache,
 		ProviderConfigCache: w.providerConfigCache,
 		ProviderInputConfig: w.Context.providerInputConfig,
 		ProviderLock:        &w.providerLock,
-		Provisioners:        w.Context.provisioners,
 		ProvisionerCache:    w.provisionerCache,
 		ProvisionerLock:     &w.provisionerLock,
 		DiffValue:           w.Context.diff,

terraform/resource.go

@@ -62,6 +62,12 @@ type InstanceInfo struct {
 
 	// Type is the resource type of this instance
 	Type string
+
+	// uniqueExtra is an internal field that can be populated to supply
+	// extra metadata that is used to identify a unique instance in
+	// the graph walk. This will be appended to HumanID when uniqueId
+	// is called.
+	uniqueExtra string
 }
 
 // HumanId is a unique Id that is human-friendly and useful for UI elements.
@@ -76,6 +82,15 @@ func (i *InstanceInfo) HumanId() string {
 		i.Id)
 }
 
+func (i *InstanceInfo) uniqueId() string {
+	prefix := i.HumanId()
+	if v := i.uniqueExtra; v != "" {
+		prefix += " " + v
+	}
+
+	return prefix
+}
+
 // ResourceConfig holds the configuration given for a resource. This is
 // done instead of a raw `map[string]interface{}` type so that rich
 // methods can be added to it to make dealing with it easier.
@@ -98,6 +113,11 @@ func NewResourceConfig(c *config.RawConfig) *ResourceConfig {
 // to modify any of the structures that are part of the resource config without
 // affecting the original configuration.
 func (c *ResourceConfig) DeepCopy() *ResourceConfig {
+	// DeepCopying a nil should return a nil to avoid panics
+	if c == nil {
+		return nil
+	}
+
 	// Copy, this will copy all the exported attributes
 	copy, err := copystructure.Config{Lock: true}.Copy(c)
 	if err != nil {
@@ -115,6 +135,11 @@ func (c *ResourceConfig) DeepCopy() *ResourceConfig {
 
 // Equal checks the equality of two resource configs.
 func (c *ResourceConfig) Equal(c2 *ResourceConfig) bool {
+	// If either are nil, then they're only equal if they're both nil
+	if c == nil || c2 == nil {
+		return c == c2
+	}
+
 	// Two resource configs if their exported properties are equal.
 	// We don't compare "raw" because it is never used again after
 	// initialization and for all intents and purposes they are equal

terraform/resource_provider.go

@@ -3,6 +3,12 @@ package terraform
 // ResourceProvider is an interface that must be implemented by any
 // resource provider: the thing that creates and manages the resources in
 // a Terraform configuration.
+//
+// Important implementation note: All returned pointers, such as
+// *ResourceConfig, *InstanceState, *InstanceDiff, etc. must not point to
+// shared data. Terraform is highly parallel and assumes that this data is safe
+// to read/write in parallel so it must be unique references. Note that it is
+// safe to return arguments as results, however.
 type ResourceProvider interface {
 	/*********************************************************************
 	* Functions related to the provider

terraform/resource_provider_mock.go

@@ -157,7 +157,7 @@ func (p *MockResourceProvider) Apply(
 		return p.ApplyFn(info, state, diff)
 	}
 
-	return p.ApplyReturn, p.ApplyReturnError
+	return p.ApplyReturn.DeepCopy(), p.ApplyReturnError
 }
 
 func (p *MockResourceProvider) Diff(
@@ -175,7 +175,7 @@ func (p *MockResourceProvider) Diff(
 		return p.DiffFn(info, state, desired)
 	}
 
-	return p.DiffReturn, p.DiffReturnError
+	return p.DiffReturn.DeepCopy(), p.DiffReturnError
 }
 
 func (p *MockResourceProvider) Refresh(
@@ -192,7 +192,7 @@ func (p *MockResourceProvider) Refresh(
 		return p.RefreshFn(info, s)
 	}
 
-	return p.RefreshReturn, p.RefreshReturnError
+	return p.RefreshReturn.DeepCopy(), p.RefreshReturnError
 }
 
 func (p *MockResourceProvider) Resources() []ResourceType {
@@ -214,7 +214,15 @@ func (p *MockResourceProvider) ImportState(info *InstanceInfo, id string) ([]*In
 		return p.ImportStateFn(info, id)
 	}
 
-	return p.ImportStateReturn, p.ImportStateReturnError
+	var result []*InstanceState
+	if p.ImportStateReturn != nil {
+		result = make([]*InstanceState, len(p.ImportStateReturn))
+		for i, v := range p.ImportStateReturn {
+			result[i] = v.DeepCopy()
+		}
+	}
+
+	return result, p.ImportStateReturnError
 }
 
 func (p *MockResourceProvider) ValidateDataSource(t string, c *ResourceConfig) ([]string, []error) {
@@ -245,7 +253,7 @@ func (p *MockResourceProvider) ReadDataDiff(
 		return p.ReadDataDiffFn(info, desired)
 	}
 
-	return p.ReadDataDiffReturn, p.ReadDataDiffReturnError
+	return p.ReadDataDiffReturn.DeepCopy(), p.ReadDataDiffReturnError
 }
 
 func (p *MockResourceProvider) ReadDataApply(
@@ -262,7 +270,7 @@ func (p *MockResourceProvider) ReadDataApply(
 		return p.ReadDataApplyFn(info, d)
 	}
 
-	return p.ReadDataApplyReturn, p.ReadDataApplyReturnError
+	return p.ReadDataApplyReturn.DeepCopy(), p.ReadDataApplyReturnError
 }
 
 func (p *MockResourceProvider) DataSources() []DataSource {

terraform/resource_test.go

@@ -239,6 +239,35 @@ func TestResourceConfigGet(t *testing.T) {
 	}
 }
 
+func TestResourceConfigDeepCopy_nil(t *testing.T) {
+	var nilRc *ResourceConfig
+	actual := nilRc.DeepCopy()
+	if actual != nil {
+		t.Fatalf("bad: %#v", actual)
+	}
+}
+
+func TestResourceConfigDeepCopy_nilComputed(t *testing.T) {
+	rc := &ResourceConfig{}
+	actual := rc.DeepCopy()
+	if actual.ComputedKeys != nil {
+		t.Fatalf("bad: %#v", actual)
+	}
+}
+
+func TestResourceConfigEqual_nil(t *testing.T) {
+	var nilRc *ResourceConfig
+	notNil := NewResourceConfig(nil)
+
+	if nilRc.Equal(notNil) {
+		t.Fatal("should not be equal")
+	}
+
+	if notNil.Equal(nilRc) {
+		t.Fatal("should not be equal")
+	}
+}
+
 func testResourceConfig(
 	t *testing.T, c map[string]interface{}) *ResourceConfig {
 	raw, err := config.NewRawConfig(c)

terraform/shadow.go

@@ -0,0 +1,28 @@
+package terraform
+
+// Shadow is the interface that any "shadow" structures must implement.
+//
+// A shadow structure is an interface implementation (typically) that
+// shadows a real implementation and verifies that the same behavior occurs
+// on both. The semantics of this behavior are up to the interface itself.
+//
+// A shadow NEVER modifies real values or state. It must always be safe to use.
+//
+// For example, a ResourceProvider shadow ensures that the same operations
+// are done on the same resources with the same configurations.
+//
+// The typical usage of a shadow following this interface is to complete
+// the real operations, then call CloseShadow which tells the shadow that
+// the real side is done. Then, once the shadow is also complete, call
+// ShadowError to find any errors that may have been caught.
+type Shadow interface {
+	// CloseShadow tells the shadow that the REAL implementation is
+	// complete. Therefore, any calls that would block should now return
+	// immediately since no more changes will happen to the real side.
+	CloseShadow() error
+
+	// ShadowError returns the errors that the shadow has found.
+	// This should be called AFTER CloseShadow and AFTER the shadow is
+	// known to be complete (no more calls to it).
+	ShadowError() error
+}

terraform/shadow_components.go

@@ -0,0 +1,265 @@
+package terraform
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/terraform/helper/shadow"
+)
+
+// newShadowComponentFactory creates a shadowed contextComponentFactory
+// so that requests to create new components result in both a real and
+// shadow side.
+func newShadowComponentFactory(
+	f contextComponentFactory) (contextComponentFactory, *shadowComponentFactory) {
+	// Create the shared data
+	shared := &shadowComponentFactoryShared{contextComponentFactory: f}
+
+	// Create the real side
+	real := &shadowComponentFactory{
+		shadowComponentFactoryShared: shared,
+	}
+
+	// Create the shadow
+	shadow := &shadowComponentFactory{
+		shadowComponentFactoryShared: shared,
+		Shadow: true,
+	}
+
+	return real, shadow
+}
+
+// shadowComponentFactory is the shadow side. Any components created
+// with this factory are fake and will not cause real work to happen.
+//
+// Unlike other shadowers, the shadow component factory will allow the
+// shadow to create _any_ component even if it is never requested on the
+// real side. This is because errors will happen later downstream as function
+// calls are made to the shadows that are never matched on the real side.
+type shadowComponentFactory struct {
+	*shadowComponentFactoryShared
+
+	Shadow bool // True if this should return the shadow
+	lock   sync.Mutex
+}
+
+func (f *shadowComponentFactory) ResourceProvider(
+	n, uid string) (ResourceProvider, error) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	real, shadow, err := f.shadowComponentFactoryShared.ResourceProvider(n, uid)
+	var result ResourceProvider = real
+	if f.Shadow {
+		result = shadow
+	}
+
+	return result, err
+}
+
+func (f *shadowComponentFactory) ResourceProvisioner(
+	n, uid string) (ResourceProvisioner, error) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	real, shadow, err := f.shadowComponentFactoryShared.ResourceProvisioner(n, uid)
+	var result ResourceProvisioner = real
+	if f.Shadow {
+		result = shadow
+	}
+
+	return result, err
+}
+
+// CloseShadow is called when the _real_ side is complete. This will cause
+// all future blocking operations to return immediately on the shadow to
+// ensure the shadow also completes.
+func (f *shadowComponentFactory) CloseShadow() error {
+	// If we aren't the shadow, just return
+	if !f.Shadow {
+		return nil
+	}
+
+	// Lock ourselves so we don't modify state
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	// Grab our shared state
+	shared := f.shadowComponentFactoryShared
+
+	// If we're already closed, its an error
+	if shared.closed {
+		return fmt.Errorf("component factory shadow already closed")
+	}
+
+	// Close all the providers and provisioners and return the error
+	var result error
+	for _, n := range shared.providerKeys {
+		_, shadow, err := shared.ResourceProvider(n, n)
+		if err == nil && shadow != nil {
+			if err := shadow.CloseShadow(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	for _, n := range shared.provisionerKeys {
+		_, shadow, err := shared.ResourceProvisioner(n, n)
+		if err == nil && shadow != nil {
+			if err := shadow.CloseShadow(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	// Mark ourselves as closed
+	shared.closed = true
+
+	return result
+}
+
+func (f *shadowComponentFactory) ShadowError() error {
+	// If we aren't the shadow, just return
+	if !f.Shadow {
+		return nil
+	}
+
+	// Lock ourselves so we don't modify state
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	// Grab our shared state
+	shared := f.shadowComponentFactoryShared
+
+	// If we're not closed, its an error
+	if !shared.closed {
+		return fmt.Errorf("component factory must be closed to retrieve errors")
+	}
+
+	// Close all the providers and provisioners and return the error
+	var result error
+	for _, n := range shared.providerKeys {
+		_, shadow, err := shared.ResourceProvider(n, n)
+		if err == nil && shadow != nil {
+			if err := shadow.ShadowError(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	for _, n := range shared.provisionerKeys {
+		_, shadow, err := shared.ResourceProvisioner(n, n)
+		if err == nil && shadow != nil {
+			if err := shadow.ShadowError(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	return result
+}
+
+// shadowComponentFactoryShared is shared data between the two factories.
+//
+// It is NOT SAFE to run any function on this struct in parallel. Lock
+// access to this struct.
+type shadowComponentFactoryShared struct {
+	contextComponentFactory
+
+	closed          bool
+	providers       shadow.KeyedValue
+	providerKeys    []string
+	provisioners    shadow.KeyedValue
+	provisionerKeys []string
+}
+
+// shadowResourceProviderFactoryEntry is the entry that is stored in
+// the Shadows key/value for a provider.
+type shadowComponentFactoryProviderEntry struct {
+	Real   ResourceProvider
+	Shadow shadowResourceProvider
+	Err    error
+}
+
+type shadowComponentFactoryProvisionerEntry struct {
+	Real   ResourceProvisioner
+	Shadow shadowResourceProvisioner
+	Err    error
+}
+
+func (f *shadowComponentFactoryShared) ResourceProvider(
+	n, uid string) (ResourceProvider, shadowResourceProvider, error) {
+	// Determine if we already have a value
+	raw, ok := f.providers.ValueOk(uid)
+	if !ok {
+		// Build the entry
+		var entry shadowComponentFactoryProviderEntry
+
+		// No value, initialize. Create the original
+		p, err := f.contextComponentFactory.ResourceProvider(n, uid)
+		if err != nil {
+			entry.Err = err
+			p = nil // Just to be sure
+		}
+
+		if p != nil {
+			// Create the shadow
+			real, shadow := newShadowResourceProvider(p)
+			entry.Real = real
+			entry.Shadow = shadow
+		}
+
+		// Store the value
+		f.providers.SetValue(uid, &entry)
+		f.providerKeys = append(f.providerKeys, uid)
+		raw = &entry
+	}
+
+	// Read the entry
+	entry, ok := raw.(*shadowComponentFactoryProviderEntry)
+	if !ok {
+		return nil, nil, fmt.Errorf("Unknown value for shadow provider: %#v", raw)
+	}
+
+	// Return
+	return entry.Real, entry.Shadow, entry.Err
+}
+
+func (f *shadowComponentFactoryShared) ResourceProvisioner(
+	n, uid string) (ResourceProvisioner, shadowResourceProvisioner, error) {
+	// Determine if we already have a value
+	raw, ok := f.provisioners.ValueOk(uid)
+	if !ok {
+		// Build the entry
+		var entry shadowComponentFactoryProvisionerEntry
+
+		// No value, initialize. Create the original
+		p, err := f.contextComponentFactory.ResourceProvisioner(n, uid)
+		if err != nil {
+			entry.Err = err
+			p = nil // Just to be sure
+		}
+
+		if p != nil {
+			// For now, just create a mock since we don't support provisioners yet
+			real, shadow := newShadowResourceProvisioner(p)
+			entry.Real = real
+			entry.Shadow = shadow
+		}
+
+		// Store the value
+		f.provisioners.SetValue(uid, &entry)
+		f.provisionerKeys = append(f.provisionerKeys, uid)
+		raw = &entry
+	}
+
+	// Read the entry
+	entry, ok := raw.(*shadowComponentFactoryProvisionerEntry)
+	if !ok {
+		return nil, nil, fmt.Errorf("Unknown value for shadow provisioner: %#v", raw)
+	}
+
+	// Return
+	return entry.Real, entry.Shadow, entry.Err
+}

terraform/shadow_context.go

@@ -0,0 +1,138 @@
+package terraform
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/mitchellh/copystructure"
+)
+
+// newShadowContext creates a new context that will shadow the given context
+// when walking the graph. The resulting context should be used _only once_
+// for a graph walk.
+//
+// The returned Shadow should be closed after the graph walk with the
+// real context is complete. Errors from the shadow can be retrieved there.
+//
+// Most importantly, any operations done on the shadow context (the returned
+// context) will NEVER affect the real context. All structures are deep
+// copied, no real providers or resources are used, etc.
+func newShadowContext(c *Context) (*Context, *Context, Shadow) {
+	// Copy the targets
+	targetRaw, err := copystructure.Copy(c.targets)
+	if err != nil {
+		panic(err)
+	}
+
+	// Copy the variables
+	varRaw, err := copystructure.Copy(c.variables)
+	if err != nil {
+		panic(err)
+	}
+
+	// Copy the provider inputs
+	providerInputRaw, err := copystructure.Copy(c.providerInputConfig)
+	if err != nil {
+		panic(err)
+	}
+
+	// The factories
+	componentsReal, componentsShadow := newShadowComponentFactory(c.components)
+
+	// Create the shadow
+	shadow := &Context{
+		components: componentsShadow,
+		destroy:    c.destroy,
+		diff:       c.diff.DeepCopy(),
+		hooks:      nil,
+		module:     c.module,
+		state:      c.state.DeepCopy(),
+		targets:    targetRaw.([]string),
+		variables:  varRaw.(map[string]interface{}),
+
+		// NOTE(mitchellh): This is not going to work for shadows that are
+		// testing that input results in the proper end state. At the time
+		// of writing, input is not used in any state-changing graph
+		// walks anyways, so this checks nothing. We set it to this to avoid
+		// any panics but even a "nil" value worked here.
+		uiInput: new(MockUIInput),
+
+		// Hardcoded to 4 since parallelism in the shadow doesn't matter
+		// a ton since we're doing far less compared to the real side
+		// and our operations are MUCH faster.
+		parallelSem:         NewSemaphore(4),
+		providerInputConfig: providerInputRaw.(map[string]map[string]interface{}),
+	}
+
+	// Create the real context. This is effectively just a copy of
+	// the context given except we need to modify some of the values
+	// to point to the real side of a shadow so the shadow can compare values.
+	real := &Context{
+		// The fields below are changed.
+		components: componentsReal,
+
+		// The fields below are direct copies
+		destroy: c.destroy,
+		diff:    c.diff,
+		// diffLock - no copy
+		hooks:  c.hooks,
+		module: c.module,
+		sh:     c.sh,
+		state:  c.state,
+		// stateLock - no copy
+		targets:   c.targets,
+		uiInput:   c.uiInput,
+		variables: c.variables,
+
+		// l - no copy
+		parallelSem:         c.parallelSem,
+		providerInputConfig: c.providerInputConfig,
+		runCh:               c.runCh,
+		shadowErr:           c.shadowErr,
+	}
+
+	return real, shadow, &shadowContextCloser{
+		Components: componentsShadow,
+	}
+}
+
+// shadowContextVerify takes the real and shadow context and verifies they
+// have equal diffs and states.
+func shadowContextVerify(real, shadow *Context) error {
+	var result error
+
+	// Compare the states
+	if !real.state.Equal(shadow.state) {
+		result = multierror.Append(result, fmt.Errorf(
+			"Real and shadow states do not match! "+
+				"Real state:\n\n%s\n\n"+
+				"Shadow state:\n\n%s\n\n",
+			real.state, shadow.state))
+	}
+
+	// Compare the diffs
+	if !real.diff.Equal(shadow.diff) {
+		result = multierror.Append(result, fmt.Errorf(
+			"Real and shadow diffs do not match! "+
+				"Real diff:\n\n%s\n\n"+
+				"Shadow diff:\n\n%s\n\n",
+			real.diff, shadow.diff))
+	}
+
+	return result
+}
+
+// shadowContextCloser is the io.Closer returned by newShadowContext that
+// closes all the shadows and returns the results.
+type shadowContextCloser struct {
+	Components *shadowComponentFactory
+}
+
+// Close closes the shadow context.
+func (c *shadowContextCloser) CloseShadow() error {
+	return c.Components.CloseShadow()
+}
+
+func (c *shadowContextCloser) ShadowError() error {
+	return c.Components.ShadowError()
+}

terraform/shadow_resource_provider.go

@@ -0,0 +1,819 @@
+package terraform
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/terraform/helper/shadow"
+)
+
+// shadowResourceProvider implements ResourceProvider for the shadow
+// eval context defined in eval_context_shadow.go.
+//
+// This is used to verify behavior with a real provider. This shouldn't
+// be used directly.
+type shadowResourceProvider interface {
+	ResourceProvider
+	Shadow
+}
+
+// newShadowResourceProvider creates a new shadowed ResourceProvider.
+//
+// This will assume a well behaved real ResourceProvider. For example,
+// it assumes that the `Resources` call underneath doesn't change values
+// since once it is called on the real provider, it will be cached and
+// returned in the shadow since number of calls to that shouldn't affect
+// actual behavior.
+//
+// However, with calls like Apply, call order is taken into account,
+// parameters are checked for equality, etc.
+func newShadowResourceProvider(p ResourceProvider) (ResourceProvider, shadowResourceProvider) {
+	// Create the shared data
+	shared := shadowResourceProviderShared{}
+
+	// Create the real provider that does actual work
+	real := &shadowResourceProviderReal{
+		ResourceProvider: p,
+		Shared:           &shared,
+	}
+
+	// Create the shadow that watches the real value
+	shadow := &shadowResourceProviderShadow{
+		Shared: &shared,
+
+		resources:   p.Resources(),
+		dataSources: p.DataSources(),
+	}
+
+	return real, shadow
+}
+
+// shadowResourceProviderReal is the real resource provider. Function calls
+// to this will perform real work. This records the parameters and return
+// values and call order for the shadow to reproduce.
+type shadowResourceProviderReal struct {
+	ResourceProvider
+
+	Shared *shadowResourceProviderShared
+}
+
+func (p *shadowResourceProviderReal) Close() error {
+	var result error
+	if c, ok := p.ResourceProvider.(ResourceProviderCloser); ok {
+		result = c.Close()
+	}
+
+	p.Shared.CloseErr.SetValue(result)
+	return result
+}
+
+func (p *shadowResourceProviderReal) Input(
+	input UIInput, c *ResourceConfig) (*ResourceConfig, error) {
+	cCopy := c.DeepCopy()
+
+	result, err := p.ResourceProvider.Input(input, c)
+	p.Shared.Input.SetValue(&shadowResourceProviderInput{
+		Config:    cCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+func (p *shadowResourceProviderReal) Validate(c *ResourceConfig) ([]string, []error) {
+	warns, errs := p.ResourceProvider.Validate(c)
+	p.Shared.Validate.SetValue(&shadowResourceProviderValidate{
+		Config:     c.DeepCopy(),
+		ResultWarn: warns,
+		ResultErr:  errs,
+	})
+
+	return warns, errs
+}
+
+func (p *shadowResourceProviderReal) Configure(c *ResourceConfig) error {
+	cCopy := c.DeepCopy()
+
+	err := p.ResourceProvider.Configure(c)
+	p.Shared.Configure.SetValue(&shadowResourceProviderConfigure{
+		Config: cCopy,
+		Result: err,
+	})
+
+	return err
+}
+
+func (p *shadowResourceProviderReal) ValidateResource(
+	t string, c *ResourceConfig) ([]string, []error) {
+	key := t
+	configCopy := c.DeepCopy()
+
+	// Real operation
+	warns, errs := p.ResourceProvider.ValidateResource(t, c)
+
+	// Initialize to ensure we always have a wrapper with a lock
+	p.Shared.ValidateResource.Init(
+		key, &shadowResourceProviderValidateResourceWrapper{})
+
+	// Get the result
+	raw := p.Shared.ValidateResource.Value(key)
+	wrapper, ok := raw.(*shadowResourceProviderValidateResourceWrapper)
+	if !ok {
+		// If this fails then we just continue with our day... the shadow
+		// will fail to but there isn't much we can do.
+		log.Printf(
+			"[ERROR] unknown value in ValidateResource shadow value: %#v", raw)
+		return warns, errs
+	}
+
+	// Lock the wrapper for writing and record our call
+	wrapper.Lock()
+	defer wrapper.Unlock()
+
+	wrapper.Calls = append(wrapper.Calls, &shadowResourceProviderValidateResource{
+		Config: configCopy,
+		Warns:  warns,
+		Errors: errs,
+	})
+
+	// Return the result
+	return warns, errs
+}
+
+func (p *shadowResourceProviderReal) Apply(
+	info *InstanceInfo,
+	state *InstanceState,
+	diff *InstanceDiff) (*InstanceState, error) {
+	// Thse have to be copied before the call since call can modify
+	stateCopy := state.DeepCopy()
+	diffCopy := diff.DeepCopy()
+
+	result, err := p.ResourceProvider.Apply(info, state, diff)
+	p.Shared.Apply.SetValue(info.uniqueId(), &shadowResourceProviderApply{
+		State:     stateCopy,
+		Diff:      diffCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+func (p *shadowResourceProviderReal) Diff(
+	info *InstanceInfo,
+	state *InstanceState,
+	desired *ResourceConfig) (*InstanceDiff, error) {
+	// Thse have to be copied before the call since call can modify
+	stateCopy := state.DeepCopy()
+	desiredCopy := desired.DeepCopy()
+
+	result, err := p.ResourceProvider.Diff(info, state, desired)
+	p.Shared.Diff.SetValue(info.uniqueId(), &shadowResourceProviderDiff{
+		State:     stateCopy,
+		Desired:   desiredCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+func (p *shadowResourceProviderReal) Refresh(
+	info *InstanceInfo,
+	state *InstanceState) (*InstanceState, error) {
+	// Thse have to be copied before the call since call can modify
+	stateCopy := state.DeepCopy()
+
+	result, err := p.ResourceProvider.Refresh(info, state)
+	p.Shared.Refresh.SetValue(info.uniqueId(), &shadowResourceProviderRefresh{
+		State:     stateCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+func (p *shadowResourceProviderReal) ValidateDataSource(
+	t string, c *ResourceConfig) ([]string, []error) {
+	key := t
+	configCopy := c.DeepCopy()
+
+	// Real operation
+	warns, errs := p.ResourceProvider.ValidateDataSource(t, c)
+
+	// Initialize
+	p.Shared.ValidateDataSource.Init(
+		key, &shadowResourceProviderValidateDataSourceWrapper{})
+
+	// Get the result
+	raw := p.Shared.ValidateDataSource.Value(key)
+	wrapper, ok := raw.(*shadowResourceProviderValidateDataSourceWrapper)
+	if !ok {
+		// If this fails then we just continue with our day... the shadow
+		// will fail to but there isn't much we can do.
+		log.Printf(
+			"[ERROR] unknown value in ValidateDataSource shadow value: %#v", raw)
+		return warns, errs
+	}
+
+	// Lock the wrapper for writing and record our call
+	wrapper.Lock()
+	defer wrapper.Unlock()
+
+	wrapper.Calls = append(wrapper.Calls, &shadowResourceProviderValidateDataSource{
+		Config: configCopy,
+		Warns:  warns,
+		Errors: errs,
+	})
+
+	// Set it
+	p.Shared.ValidateDataSource.SetValue(key, wrapper)
+
+	// Return the result
+	return warns, errs
+}
+
+func (p *shadowResourceProviderReal) ReadDataDiff(
+	info *InstanceInfo,
+	desired *ResourceConfig) (*InstanceDiff, error) {
+	// These have to be copied before the call since call can modify
+	desiredCopy := desired.DeepCopy()
+
+	result, err := p.ResourceProvider.ReadDataDiff(info, desired)
+	p.Shared.ReadDataDiff.SetValue(info.uniqueId(), &shadowResourceProviderReadDataDiff{
+		Desired:   desiredCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+func (p *shadowResourceProviderReal) ReadDataApply(
+	info *InstanceInfo,
+	diff *InstanceDiff) (*InstanceState, error) {
+	// Thse have to be copied before the call since call can modify
+	diffCopy := diff.DeepCopy()
+
+	result, err := p.ResourceProvider.ReadDataApply(info, diff)
+	p.Shared.ReadDataApply.SetValue(info.uniqueId(), &shadowResourceProviderReadDataApply{
+		Diff:      diffCopy,
+		Result:    result.DeepCopy(),
+		ResultErr: err,
+	})
+
+	return result, err
+}
+
+// shadowResourceProviderShadow is the shadow resource provider. Function
+// calls never affect real resources. This is paired with the "real" side
+// which must be called properly to enable recording.
+type shadowResourceProviderShadow struct {
+	Shared *shadowResourceProviderShared
+
+	// Cached values that are expected to not change
+	resources   []ResourceType
+	dataSources []DataSource
+
+	Error     error // Error is the list of errors from the shadow
+	ErrorLock sync.Mutex
+}
+
+type shadowResourceProviderShared struct {
+	// NOTE: Anytime a value is added here, be sure to add it to
+	// the Close() method so that it is closed.
+
+	CloseErr           shadow.Value
+	Input              shadow.Value
+	Validate           shadow.Value
+	Configure          shadow.Value
+	ValidateResource   shadow.KeyedValue
+	Apply              shadow.KeyedValue
+	Diff               shadow.KeyedValue
+	Refresh            shadow.KeyedValue
+	ValidateDataSource shadow.KeyedValue
+	ReadDataDiff       shadow.KeyedValue
+	ReadDataApply      shadow.KeyedValue
+}
+
+func (p *shadowResourceProviderShared) Close() error {
+	closers := []io.Closer{
+		&p.CloseErr, &p.Input, &p.Validate,
+		&p.Configure, &p.ValidateResource, &p.Apply, &p.Diff,
+		&p.Refresh, &p.ValidateDataSource, &p.ReadDataDiff,
+	}
+
+	for _, c := range closers {
+		// This should never happen, but we don't panic because a panic
+		// could affect the real behavior of Terraform and a shadow should
+		// never be able to do that.
+		if err := c.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *shadowResourceProviderShadow) CloseShadow() error {
+	err := p.Shared.Close()
+	if err != nil {
+		err = fmt.Errorf("close error: %s", err)
+	}
+
+	return err
+}
+
+func (p *shadowResourceProviderShadow) ShadowError() error {
+	return p.Error
+}
+
+func (p *shadowResourceProviderShadow) Resources() []ResourceType {
+	return p.resources
+}
+
+func (p *shadowResourceProviderShadow) DataSources() []DataSource {
+	return p.dataSources
+}
+
+func (p *shadowResourceProviderShadow) Close() error {
+	v := p.Shared.CloseErr.Value()
+	if v == nil {
+		return nil
+	}
+
+	return v.(error)
+}
+
+func (p *shadowResourceProviderShadow) Input(
+	input UIInput, c *ResourceConfig) (*ResourceConfig, error) {
+	// Get the result of the input call
+	raw := p.Shared.Input.Value()
+	if raw == nil {
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderInput)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'input' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !c.Equal(result.Config) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Input had unequal configurations (real, then shadow):\n\n%#v\n\n%#v",
+			result.Config, c))
+		p.ErrorLock.Unlock()
+	}
+
+	// Return the results
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) Validate(c *ResourceConfig) ([]string, []error) {
+	// Get the result of the validate call
+	raw := p.Shared.Validate.Value()
+	if raw == nil {
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderValidate)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'validate' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !c.Equal(result.Config) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Validate had unequal configurations (real, then shadow):\n\n%#v\n\n%#v",
+			result.Config, c))
+		p.ErrorLock.Unlock()
+	}
+
+	// Return the results
+	return result.ResultWarn, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) Configure(c *ResourceConfig) error {
+	// Get the result of the call
+	raw := p.Shared.Configure.Value()
+	if raw == nil {
+		return nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderConfigure)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'configure' shadow value: %#v", raw))
+		return nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !c.Equal(result.Config) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Configure had unequal configurations (real, then shadow):\n\n%#v\n\n%#v",
+			result.Config, c))
+		p.ErrorLock.Unlock()
+	}
+
+	// Return the results
+	return result.Result
+}
+
+func (p *shadowResourceProviderShadow) ValidateResource(t string, c *ResourceConfig) ([]string, []error) {
+	// Unique key
+	key := t
+
+	// Get the initial value
+	raw := p.Shared.ValidateResource.Value(key)
+
+	// Find a validation with our configuration
+	var result *shadowResourceProviderValidateResource
+	for {
+		// Get the value
+		if raw == nil {
+			p.ErrorLock.Lock()
+			defer p.ErrorLock.Unlock()
+			p.Error = multierror.Append(p.Error, fmt.Errorf(
+				"Unknown 'ValidateResource' call for %q:\n\n%#v",
+				key, c))
+			return nil, nil
+		}
+
+		wrapper, ok := raw.(*shadowResourceProviderValidateResourceWrapper)
+		if !ok {
+			p.ErrorLock.Lock()
+			defer p.ErrorLock.Unlock()
+			p.Error = multierror.Append(p.Error, fmt.Errorf(
+				"Unknown 'ValidateResource' shadow value: %#v", raw))
+			return nil, nil
+		}
+
+		// Look for the matching call with our configuration
+		wrapper.RLock()
+		for _, call := range wrapper.Calls {
+			if call.Config.Equal(c) {
+				result = call
+				break
+			}
+		}
+		wrapper.RUnlock()
+
+		// If we found a result, exit
+		if result != nil {
+			break
+		}
+
+		// Wait for a change so we can get the wrapper again
+		raw = p.Shared.ValidateResource.WaitForChange(key)
+	}
+
+	return result.Warns, result.Errors
+}
+
+func (p *shadowResourceProviderShadow) Apply(
+	info *InstanceInfo,
+	state *InstanceState,
+	diff *InstanceDiff) (*InstanceState, error) {
+	// Unique key
+	key := info.uniqueId()
+	raw := p.Shared.Apply.Value(key)
+	if raw == nil {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'apply' call for %q:\n\n%#v\n\n%#v",
+			key, state, diff))
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderApply)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'apply' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !state.Equal(result.State) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Apply: state had unequal states (real, then shadow):\n\n%#v\n\n%#v",
+			result.State, state))
+		p.ErrorLock.Unlock()
+	}
+
+	if !diff.Equal(result.Diff) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Apply: unequal diffs (real, then shadow):\n\n%#v\n\n%#v",
+			result.Diff, diff))
+		p.ErrorLock.Unlock()
+	}
+
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) Diff(
+	info *InstanceInfo,
+	state *InstanceState,
+	desired *ResourceConfig) (*InstanceDiff, error) {
+	// Unique key
+	key := info.uniqueId()
+	raw := p.Shared.Diff.Value(key)
+	if raw == nil {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'diff' call for %q:\n\n%#v\n\n%#v",
+			key, state, desired))
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderDiff)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'diff' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !state.Equal(result.State) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Diff %q had unequal states (real, then shadow):\n\n%#v\n\n%#v",
+			key, result.State, state))
+		p.ErrorLock.Unlock()
+	}
+	if !desired.Equal(result.Desired) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Diff %q had unequal states (real, then shadow):\n\n%#v\n\n%#v",
+			key, result.Desired, desired))
+		p.ErrorLock.Unlock()
+	}
+
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) Refresh(
+	info *InstanceInfo,
+	state *InstanceState) (*InstanceState, error) {
+	// Unique key
+	key := info.uniqueId()
+	raw := p.Shared.Refresh.Value(key)
+	if raw == nil {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'refresh' call for %q:\n\n%#v",
+			key, state))
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderRefresh)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'refresh' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !state.Equal(result.State) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Refresh %q had unequal states (real, then shadow):\n\n%#v\n\n%#v",
+			key, result.State, state))
+		p.ErrorLock.Unlock()
+	}
+
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) ValidateDataSource(
+	t string, c *ResourceConfig) ([]string, []error) {
+	// Unique key
+	key := t
+
+	// Get the initial value
+	raw := p.Shared.ValidateDataSource.Value(key)
+
+	// Find a validation with our configuration
+	var result *shadowResourceProviderValidateDataSource
+	for {
+		// Get the value
+		if raw == nil {
+			p.ErrorLock.Lock()
+			defer p.ErrorLock.Unlock()
+			p.Error = multierror.Append(p.Error, fmt.Errorf(
+				"Unknown 'ValidateDataSource' call for %q:\n\n%#v",
+				key, c))
+			return nil, nil
+		}
+
+		wrapper, ok := raw.(*shadowResourceProviderValidateDataSourceWrapper)
+		if !ok {
+			p.ErrorLock.Lock()
+			defer p.ErrorLock.Unlock()
+			p.Error = multierror.Append(p.Error, fmt.Errorf(
+				"Unknown 'ValidateDataSource' shadow value: %#v", raw))
+			return nil, nil
+		}
+
+		// Look for the matching call with our configuration
+		wrapper.RLock()
+		for _, call := range wrapper.Calls {
+			if call.Config.Equal(c) {
+				result = call
+				break
+			}
+		}
+		wrapper.RUnlock()
+
+		// If we found a result, exit
+		if result != nil {
+			break
+		}
+
+		// Wait for a change so we can get the wrapper again
+		raw = p.Shared.ValidateDataSource.WaitForChange(key)
+	}
+
+	return result.Warns, result.Errors
+}
+
+func (p *shadowResourceProviderShadow) ReadDataDiff(
+	info *InstanceInfo,
+	desired *ResourceConfig) (*InstanceDiff, error) {
+	// Unique key
+	key := info.uniqueId()
+	raw := p.Shared.ReadDataDiff.Value(key)
+	if raw == nil {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'ReadDataDiff' call for %q:\n\n%#v",
+			key, desired))
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderReadDataDiff)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'ReadDataDiff' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !desired.Equal(result.Desired) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"ReadDataDiff %q had unequal configs (real, then shadow):\n\n%#v\n\n%#v",
+			key, result.Desired, desired))
+		p.ErrorLock.Unlock()
+	}
+
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) ReadDataApply(
+	info *InstanceInfo,
+	d *InstanceDiff) (*InstanceState, error) {
+	// Unique key
+	key := info.uniqueId()
+	raw := p.Shared.ReadDataApply.Value(key)
+	if raw == nil {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'ReadDataApply' call for %q:\n\n%#v",
+			key, d))
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProviderReadDataApply)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'ReadDataApply' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// Compare the parameters, which should be identical
+	if !d.Equal(result.Diff) {
+		p.ErrorLock.Lock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"ReadDataApply: unequal diffs (real, then shadow):\n\n%#v\n\n%#v",
+			result.Diff, d))
+		p.ErrorLock.Unlock()
+	}
+
+	return result.Result, result.ResultErr
+}
+
+func (p *shadowResourceProviderShadow) ImportState(info *InstanceInfo, id string) ([]*InstanceState, error) {
+	panic("import not supported by shadow graph")
+}
+
+// The structs for the various function calls are put below. These structs
+// are used to carry call information across the real/shadow boundaries.
+
+type shadowResourceProviderInput struct {
+	Config    *ResourceConfig
+	Result    *ResourceConfig
+	ResultErr error
+}
+
+type shadowResourceProviderValidate struct {
+	Config     *ResourceConfig
+	ResultWarn []string
+	ResultErr  []error
+}
+
+type shadowResourceProviderConfigure struct {
+	Config *ResourceConfig
+	Result error
+}
+
+type shadowResourceProviderValidateResourceWrapper struct {
+	sync.RWMutex
+
+	Calls []*shadowResourceProviderValidateResource
+}
+
+type shadowResourceProviderValidateResource struct {
+	Config *ResourceConfig
+	Warns  []string
+	Errors []error
+}
+
+type shadowResourceProviderApply struct {
+	State     *InstanceState
+	Diff      *InstanceDiff
+	Result    *InstanceState
+	ResultErr error
+}
+
+type shadowResourceProviderDiff struct {
+	State     *InstanceState
+	Desired   *ResourceConfig
+	Result    *InstanceDiff
+	ResultErr error
+}
+
+type shadowResourceProviderRefresh struct {
+	State     *InstanceState
+	Result    *InstanceState
+	ResultErr error
+}
+
+type shadowResourceProviderValidateDataSourceWrapper struct {
+	sync.RWMutex
+
+	Calls []*shadowResourceProviderValidateDataSource
+}
+
+type shadowResourceProviderValidateDataSource struct {
+	Config *ResourceConfig
+	Warns  []string
+	Errors []error
+}
+
+type shadowResourceProviderReadDataDiff struct {
+	Desired   *ResourceConfig
+	Result    *InstanceDiff
+	ResultErr error
+}
+
+type shadowResourceProviderReadDataApply struct {
+	Diff      *InstanceDiff
+	Result    *InstanceState
+	ResultErr error
+}

terraform/shadow_resource_provider_test.go

@@ -0,0 +1,531 @@
+package terraform
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestShadowResourceProvider_impl(t *testing.T) {
+	var _ Shadow = new(shadowResourceProviderShadow)
+}
+
+func TestShadowResourceProvider_cachedValues(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Resources
+	{
+		actual := shadow.Resources()
+		expected := real.Resources()
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("bad:\n\n%#v\n\n%#v", actual, expected)
+		}
+	}
+
+	// DataSources
+	{
+		actual := shadow.DataSources()
+		expected := real.DataSources()
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("bad:\n\n%#v\n\n%#v", actual, expected)
+		}
+	}
+}
+
+func TestShadowResourceProviderInput(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	ui := new(MockUIInput)
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	returnConfig := testResourceConfig(t, map[string]interface{}{
+		"bar": "baz",
+	})
+
+	// Configure the mock
+	mock.InputReturnConfig = returnConfig
+
+	// Verify that it blocks until the real input is called
+	var actual *ResourceConfig
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		actual, err = shadow.Input(ui, config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real input
+	realResult, realErr := real.Input(ui, config)
+	if !realResult.Equal(returnConfig) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %s", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !actual.Equal(returnConfig) {
+		t.Fatalf("bad: %#v", actual)
+	}
+	if err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderInput_badInput(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	ui := new(MockUIInput)
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	configBad := testResourceConfig(t, map[string]interface{}{
+		"foo": "nope",
+	})
+
+	// Call the real with one
+	real.Input(ui, config)
+
+	// Call the shadow with another
+	_, err := shadow.Input(ui, configBad)
+	if err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+
+	// Verify we have an error
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err == nil {
+		t.Fatal("should error")
+	}
+}
+
+func TestShadowResourceProviderValidate(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	returnWarns := []string{"foo"}
+	returnErrs := []error{fmt.Errorf("bar")}
+
+	// Configure the mock
+	mock.ValidateReturnWarns = returnWarns
+	mock.ValidateReturnErrors = returnErrs
+
+	// Verify that it blocks until the real func is called
+	var warns []string
+	var errs []error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		warns, errs = shadow.Validate(config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realWarns, realErrs := real.Validate(config)
+	if !reflect.DeepEqual(realWarns, returnWarns) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+	if !reflect.DeepEqual(realErrs, returnErrs) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !reflect.DeepEqual(warns, returnWarns) {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if !reflect.DeepEqual(errs, returnErrs) {
+		t.Fatalf("bad: %#v", errs)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderValidate_badInput(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	configBad := testResourceConfig(t, map[string]interface{}{
+		"foo": "nope",
+	})
+
+	// Call the real with one
+	real.Validate(config)
+
+	// Call the shadow with another
+	shadow.Validate(configBad)
+
+	// Verify we have an error
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err == nil {
+		t.Fatal("should error")
+	}
+}
+
+func TestShadowResourceProviderConfigure(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	returnErr := fmt.Errorf("bar")
+
+	// Configure the mock
+	mock.ConfigureReturnError = returnErr
+
+	// Verify that it blocks until the real func is called
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		err = shadow.Configure(config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realErr := real.Configure(config)
+	if !reflect.DeepEqual(realErr, returnErr) {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !reflect.DeepEqual(err, returnErr) {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderConfigure_badInput(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	configBad := testResourceConfig(t, map[string]interface{}{
+		"foo": "nope",
+	})
+
+	// Call the real with one
+	real.Configure(config)
+
+	// Call the shadow with another
+	shadow.Configure(configBad)
+
+	// Verify we have an error
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err == nil {
+		t.Fatal("should error")
+	}
+}
+
+func TestShadowResourceProviderApply(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	info := &InstanceInfo{Id: "foo"}
+	state := &InstanceState{ID: "foo"}
+	diff := &InstanceDiff{Destroy: true}
+	mockResult := &InstanceState{ID: "bar"}
+
+	// Configure the mock
+	mock.ApplyReturn = mockResult
+
+	// Verify that it blocks until the real func is called
+	var result *InstanceState
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		result, err = shadow.Apply(info, state, diff)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realResult, realErr := real.Apply(info, state, diff)
+	if !realResult.Equal(mockResult) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !result.Equal(mockResult) {
+		t.Fatalf("bad: %#v", result)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderApply_modifyDiff(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	info := &InstanceInfo{Id: "foo"}
+	state := &InstanceState{ID: "foo"}
+	diff := &InstanceDiff{}
+	mockResult := &InstanceState{ID: "foo"}
+
+	// Configure the mock
+	mock.ApplyFn = func(
+		info *InstanceInfo,
+		s *InstanceState, d *InstanceDiff) (*InstanceState, error) {
+		d.Destroy = true
+		return s, nil
+	}
+
+	// Call the real func
+	realResult, realErr := real.Apply(info, state.DeepCopy(), diff.DeepCopy())
+	if !realResult.Equal(mockResult) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// Verify the shadow returned the same values
+	result, err := shadow.Apply(info, state.DeepCopy(), diff.DeepCopy())
+	if !result.Equal(mockResult) {
+		t.Fatalf("bad: %#v", result)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderApply_modifyState(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	info := &InstanceInfo{Id: "foo"}
+	state := &InstanceState{ID: ""}
+	diff := &InstanceDiff{}
+	mockResult := &InstanceState{ID: "foo"}
+
+	// Configure the mock
+	mock.ApplyFn = func(
+		info *InstanceInfo,
+		s *InstanceState, d *InstanceDiff) (*InstanceState, error) {
+		s.ID = "foo"
+		return s, nil
+	}
+
+	// Call the real func
+	realResult, realErr := real.Apply(info, state.DeepCopy(), diff)
+	if !realResult.Equal(mockResult) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// Verify the shadow returned the same values
+	result, err := shadow.Apply(info, state.DeepCopy(), diff)
+	if !result.Equal(mockResult) {
+		t.Fatalf("bad: %#v", result)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderDiff(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	info := &InstanceInfo{Id: "foo"}
+	state := &InstanceState{ID: "foo"}
+	desired := testResourceConfig(t, map[string]interface{}{"foo": "bar"})
+	mockResult := &InstanceDiff{Destroy: true}
+
+	// Configure the mock
+	mock.DiffReturn = mockResult
+
+	// Verify that it blocks until the real func is called
+	var result *InstanceDiff
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		result, err = shadow.Diff(info, state, desired)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realResult, realErr := real.Diff(info, state, desired)
+	if !reflect.DeepEqual(realResult, mockResult) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !reflect.DeepEqual(result, mockResult) {
+		t.Fatalf("bad: %#v", result)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProviderRefresh(t *testing.T) {
+	mock := new(MockResourceProvider)
+	real, shadow := newShadowResourceProvider(mock)
+
+	// Test values
+	info := &InstanceInfo{Id: "foo"}
+	state := &InstanceState{ID: "foo"}
+	mockResult := &InstanceState{ID: "bar"}
+
+	// Configure the mock
+	mock.RefreshReturn = mockResult
+
+	// Verify that it blocks until the real func is called
+	var result *InstanceState
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		result, err = shadow.Refresh(info, state)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realResult, realErr := real.Refresh(info, state)
+	if !realResult.Equal(mockResult) {
+		t.Fatalf("bad: %#v", realResult)
+	}
+	if realErr != nil {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !result.Equal(mockResult) {
+		t.Fatalf("bad: %#v", result)
+	}
+	if err != nil {
+		t.Fatalf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}

terraform/shadow_resource_provisioner.go

@@ -0,0 +1,271 @@
+package terraform
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"sync"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/terraform/helper/shadow"
+)
+
+// shadowResourceProvisioner implements ResourceProvisioner for the shadow
+// eval context defined in eval_context_shadow.go.
+//
+// This is used to verify behavior with a real provisioner. This shouldn't
+// be used directly.
+type shadowResourceProvisioner interface {
+	ResourceProvisioner
+	Shadow
+}
+
+// newShadowResourceProvisioner creates a new shadowed ResourceProvisioner.
+func newShadowResourceProvisioner(
+	p ResourceProvisioner) (ResourceProvisioner, shadowResourceProvisioner) {
+	// Create the shared data
+	shared := shadowResourceProvisionerShared{
+		Validate: shadow.ComparedValue{
+			Func: shadowResourceProvisionerValidateCompare,
+		},
+	}
+
+	// Create the real provisioner that does actual work
+	real := &shadowResourceProvisionerReal{
+		ResourceProvisioner: p,
+		Shared:              &shared,
+	}
+
+	// Create the shadow that watches the real value
+	shadow := &shadowResourceProvisionerShadow{
+		Shared: &shared,
+	}
+
+	return real, shadow
+}
+
+// shadowResourceProvisionerReal is the real resource provisioner. Function calls
+// to this will perform real work. This records the parameters and return
+// values and call order for the shadow to reproduce.
+type shadowResourceProvisionerReal struct {
+	ResourceProvisioner
+
+	Shared *shadowResourceProvisionerShared
+}
+
+func (p *shadowResourceProvisionerReal) Close() error {
+	var result error
+	if c, ok := p.ResourceProvisioner.(ResourceProvisionerCloser); ok {
+		result = c.Close()
+	}
+
+	p.Shared.CloseErr.SetValue(result)
+	return result
+}
+
+func (p *shadowResourceProvisionerReal) Validate(c *ResourceConfig) ([]string, []error) {
+	warns, errs := p.ResourceProvisioner.Validate(c)
+	p.Shared.Validate.SetValue(&shadowResourceProvisionerValidate{
+		Config:     c,
+		ResultWarn: warns,
+		ResultErr:  errs,
+	})
+
+	return warns, errs
+}
+
+func (p *shadowResourceProvisionerReal) Apply(
+	output UIOutput, s *InstanceState, c *ResourceConfig) error {
+	err := p.ResourceProvisioner.Apply(output, s, c)
+
+	// Write the result, grab a lock for writing. This should nver
+	// block long since the operations below don't block.
+	p.Shared.ApplyLock.Lock()
+	defer p.Shared.ApplyLock.Unlock()
+
+	key := s.ID
+	raw, ok := p.Shared.Apply.ValueOk(key)
+	if !ok {
+		// Setup a new value
+		raw = &shadow.ComparedValue{
+			Func: shadowResourceProvisionerApplyCompare,
+		}
+
+		// Set it
+		p.Shared.Apply.SetValue(key, raw)
+	}
+
+	compareVal, ok := raw.(*shadow.ComparedValue)
+	if !ok {
+		// Just log and return so that we don't cause the real side
+		// any side effects.
+		log.Printf("[ERROR] unknown value in 'apply': %#v", raw)
+		return err
+	}
+
+	// Write the resulting value
+	compareVal.SetValue(&shadowResourceProvisionerApply{
+		Config:    c,
+		ResultErr: err,
+	})
+
+	return err
+}
+
+// shadowResourceProvisionerShadow is the shadow resource provisioner. Function
+// calls never affect real resources. This is paired with the "real" side
+// which must be called properly to enable recording.
+type shadowResourceProvisionerShadow struct {
+	Shared *shadowResourceProvisionerShared
+
+	Error     error // Error is the list of errors from the shadow
+	ErrorLock sync.Mutex
+}
+
+type shadowResourceProvisionerShared struct {
+	// NOTE: Anytime a value is added here, be sure to add it to
+	// the Close() method so that it is closed.
+
+	CloseErr  shadow.Value
+	Validate  shadow.ComparedValue
+	Apply     shadow.KeyedValue
+	ApplyLock sync.Mutex // For writing only
+}
+
+func (p *shadowResourceProvisionerShared) Close() error {
+	closers := []io.Closer{
+		&p.CloseErr,
+	}
+
+	for _, c := range closers {
+		// This should never happen, but we don't panic because a panic
+		// could affect the real behavior of Terraform and a shadow should
+		// never be able to do that.
+		if err := c.Close(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *shadowResourceProvisionerShadow) CloseShadow() error {
+	err := p.Shared.Close()
+	if err != nil {
+		err = fmt.Errorf("close error: %s", err)
+	}
+
+	return err
+}
+
+func (p *shadowResourceProvisionerShadow) ShadowError() error {
+	return p.Error
+}
+
+func (p *shadowResourceProvisionerShadow) Close() error {
+	v := p.Shared.CloseErr.Value()
+	if v == nil {
+		return nil
+	}
+
+	return v.(error)
+}
+
+func (p *shadowResourceProvisionerShadow) Validate(c *ResourceConfig) ([]string, []error) {
+	// Get the result of the validate call
+	raw := p.Shared.Validate.Value(c)
+	if raw == nil {
+		return nil, nil
+	}
+
+	result, ok := raw.(*shadowResourceProvisionerValidate)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'validate' shadow value: %#v", raw))
+		return nil, nil
+	}
+
+	// We don't need to compare configurations because we key on the
+	// configuration so just return right away.
+	return result.ResultWarn, result.ResultErr
+}
+
+func (p *shadowResourceProvisionerShadow) Apply(
+	output UIOutput, s *InstanceState, c *ResourceConfig) error {
+	// Get the value based on the key
+	key := s.ID
+	raw := p.Shared.Apply.Value(key)
+	if raw == nil {
+		return nil
+	}
+
+	compareVal, ok := raw.(*shadow.ComparedValue)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'apply' shadow value: %#v", raw))
+		return nil
+	}
+
+	// With the compared value, we compare against our config
+	raw = compareVal.Value(c)
+	if raw == nil {
+		return nil
+	}
+
+	result, ok := raw.(*shadowResourceProvisionerApply)
+	if !ok {
+		p.ErrorLock.Lock()
+		defer p.ErrorLock.Unlock()
+		p.Error = multierror.Append(p.Error, fmt.Errorf(
+			"Unknown 'apply' shadow value: %#v", raw))
+		return nil
+	}
+
+	return result.ResultErr
+}
+
+// The structs for the various function calls are put below. These structs
+// are used to carry call information across the real/shadow boundaries.
+
+type shadowResourceProvisionerValidate struct {
+	Config     *ResourceConfig
+	ResultWarn []string
+	ResultErr  []error
+}
+
+type shadowResourceProvisionerApply struct {
+	Config    *ResourceConfig
+	ResultErr error
+}
+
+func shadowResourceProvisionerValidateCompare(k, v interface{}) bool {
+	c, ok := k.(*ResourceConfig)
+	if !ok {
+		return false
+	}
+
+	result, ok := v.(*shadowResourceProvisionerValidate)
+	if !ok {
+		return false
+	}
+
+	return c.Equal(result.Config)
+}
+
+func shadowResourceProvisionerApplyCompare(k, v interface{}) bool {
+	c, ok := k.(*ResourceConfig)
+	if !ok {
+		return false
+	}
+
+	result, ok := v.(*shadowResourceProvisionerApply)
+	if !ok {
+		return false
+	}
+
+	return c.Equal(result.Config)
+}

terraform/shadow_resource_provisioner_test.go

@@ -0,0 +1,178 @@
+package terraform
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestShadowResourceProvisioner_impl(t *testing.T) {
+	var _ Shadow = new(shadowResourceProvisionerShadow)
+}
+
+func TestShadowResourceProvisionerValidate(t *testing.T) {
+	mock := new(MockResourceProvisioner)
+	real, shadow := newShadowResourceProvisioner(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	returnWarns := []string{"foo"}
+	returnErrs := []error{fmt.Errorf("bar")}
+
+	// Configure the mock
+	mock.ValidateReturnWarns = returnWarns
+	mock.ValidateReturnErrors = returnErrs
+
+	// Verify that it blocks until the real func is called
+	var warns []string
+	var errs []error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		warns, errs = shadow.Validate(config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realWarns, realErrs := real.Validate(config)
+	if !reflect.DeepEqual(realWarns, returnWarns) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+	if !reflect.DeepEqual(realErrs, returnErrs) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !reflect.DeepEqual(warns, returnWarns) {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if !reflect.DeepEqual(errs, returnErrs) {
+		t.Fatalf("bad: %#v", errs)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProvisionerValidate_diff(t *testing.T) {
+	mock := new(MockResourceProvisioner)
+	real, shadow := newShadowResourceProvisioner(mock)
+
+	// Test values
+	config := testResourceConfig(t, map[string]interface{}{
+		"foo": "bar",
+	})
+	returnWarns := []string{"foo"}
+	returnErrs := []error{fmt.Errorf("bar")}
+
+	// Configure the mock
+	mock.ValidateReturnWarns = returnWarns
+	mock.ValidateReturnErrors = returnErrs
+
+	// Run a real validation with a config
+	real.Validate(testResourceConfig(t, map[string]interface{}{"bar": "baz"}))
+
+	// Verify that it blocks until the real func is called
+	var warns []string
+	var errs []error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		warns, errs = shadow.Validate(config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realWarns, realErrs := real.Validate(config)
+	if !reflect.DeepEqual(realWarns, returnWarns) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+	if !reflect.DeepEqual(realErrs, returnErrs) {
+		t.Fatalf("bad: %#v", realWarns)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if !reflect.DeepEqual(warns, returnWarns) {
+		t.Fatalf("bad: %#v", warns)
+	}
+	if !reflect.DeepEqual(errs, returnErrs) {
+		t.Fatalf("bad: %#v", errs)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}
+
+func TestShadowResourceProvisionerApply(t *testing.T) {
+	mock := new(MockResourceProvisioner)
+	real, shadow := newShadowResourceProvisioner(mock)
+
+	// Test values
+	output := new(MockUIOutput)
+	state := &InstanceState{ID: "foo"}
+	config := testResourceConfig(t, map[string]interface{}{"foo": "bar"})
+	mockReturn := errors.New("err")
+
+	// Configure the mock
+	mock.ApplyReturnError = mockReturn
+
+	// Verify that it blocks until the real func is called
+	var err error
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		err = shadow.Apply(output, state, config)
+	}()
+
+	select {
+	case <-doneCh:
+		t.Fatal("should block until finished")
+	case <-time.After(10 * time.Millisecond):
+	}
+
+	// Call the real func
+	realErr := real.Apply(output, state, config)
+	if realErr != mockReturn {
+		t.Fatalf("bad: %#v", realErr)
+	}
+
+	// The shadow should finish now
+	<-doneCh
+
+	// Verify the shadow returned the same values
+	if err != mockReturn {
+		t.Errorf("bad: %#v", err)
+	}
+
+	// Verify we have no errors
+	if err := shadow.CloseShadow(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+	if err := shadow.ShadowError(); err != nil {
+		t.Fatalf("bad: %s", err)
+	}
+}

terraform/terraform_test.go

@@ -23,6 +23,7 @@ const fixtureDir = "./test-fixtures"
 
 func TestMain(m *testing.M) {
 	flag.Parse()
+
 	if testing.Verbose() {
 		// if we're verbose, use the logging requested by TF_LOG
 		logging.SetOutput()
@@ -31,6 +32,12 @@ func TestMain(m *testing.M) {
 		log.SetOutput(ioutil.Discard)
 	}
 
+	// Make sure shadow operations fail our real tests
+	contextFailOnShadowError = true
+
+	// Always DeepCopy the Diff on every Plan during a test
+	contextTestDeepCopyOnPlan = true
+
 	os.Exit(m.Run())
 }
 

terraform/transform_deposed.go

@@ -72,7 +72,7 @@ func (n *graphNodeDeposedResource) EvalTree() EvalNode {
 	seq := &EvalSequence{Nodes: make([]EvalNode, 0, 5)}
 
 	// Build instance info
-	info := &InstanceInfo{Id: n.ResourceName, Type: n.ResourceType}
+	info := &InstanceInfo{Id: n.Name(), Type: n.ResourceType}
 	seq.Nodes = append(seq.Nodes, &EvalInstanceInfo{Info: info})
 
 	// Refresh the resource

terraform/transform_resource.go

@@ -862,6 +862,7 @@ func (n *graphNodeExpandedResourceDestroy) ConfigType() GraphNodeConfigType {
 // GraphNodeEvalable impl.
 func (n *graphNodeExpandedResourceDestroy) EvalTree() EvalNode {
 	info := n.instanceInfo()
+	info.uniqueExtra = "destroy"
 
 	var diffApply *InstanceDiff
 	var provider ResourceProvider