provisioners/remote-exec: listen to Stop

builtin/provisioners/remote-exec/resource_provisioner.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/terraform/communicator"
@@ -68,7 +69,7 @@ func applyFn(ctx context.Context) error {
 	}
 
 	// Copy and execute each script
-	if err := runScripts(o, comm, scripts); err != nil {
+	if err := runScripts(ctx, o, comm, scripts); err != nil {
 		return err
 	}
 
@@ -133,18 +134,29 @@ func collectScripts(d *schema.ResourceData) ([]io.ReadCloser, error) {
 
 // runScripts is used to copy and execute a set of scripts
 func runScripts(
+	ctx context.Context,
 	o terraform.UIOutput,
 	comm communicator.Communicator,
 	scripts []io.ReadCloser) error {
+	// Wrap out context in a cancelation function that we use to
+	// kill the connection.
+	ctx, cancelFunc := context.WithCancel(ctx)
+	defer cancelFunc()
+
+	// Wait for the context to end and then disconnect
+	go func() {
+		<-ctx.Done()
+		comm.Disconnect()
+	}()
+
 	// Wait and retry until we establish the connection
-	err := retryFunc(comm.Timeout(), func() error {
+	err := retryFunc(ctx, comm.Timeout(), func() error {
 		err := comm.Connect(o)
 		return err
 	})
 	if err != nil {
 		return err
 	}
-	defer comm.Disconnect()
 
 	for _, script := range scripts {
 		var cmd *remote.Cmd
@@ -156,7 +168,7 @@ func runScripts(
 		go copyOutput(o, errR, errDoneCh)
 
 		remotePath := comm.ScriptPath()
-		err = retryFunc(comm.Timeout(), func() error {
+		err = retryFunc(ctx, comm.Timeout(), func() error {
 			if err := comm.UploadScript(remotePath, script); err != nil {
 				return fmt.Errorf("Failed to upload script: %v", err)
 			}
@@ -179,6 +191,13 @@ func runScripts(
 			}
 		}
 
+		// If we have an error, end our context so the disconnect happens.
+		// This has to happen before the output cleanup below since during
+		// an interrupt this will cause the outputs to end.
+		if err != nil {
+			cancelFunc()
+		}
+
 		// Wait for output to clean up
 		outW.Close()
 		errW.Close()
@@ -212,19 +231,54 @@ func copyOutput(
 }
 
 // retryFunc is used to retry a function for a given duration
-func retryFunc(timeout time.Duration, f func() error) error {
-	finish := time.After(timeout)
-	for {
-		err := f()
-		if err == nil {
-			return nil
-		}
-		log.Printf("Retryable error: %v", err)
+func retryFunc(ctx context.Context, timeout time.Duration, f func() error) error {
+	// Build a new context with the timeout
+	ctx, done := context.WithTimeout(ctx, timeout)
+	defer done()
 
-		select {
-		case <-finish:
-			return err
-		case <-time.After(3 * time.Second):
+	// Try the function in a goroutine
+	var errVal atomic.Value
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+
+		for {
+			// If our context ended, we want to exit right away.
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			// Try the function call
+			err := f()
+			if err == nil {
+				return
+			}
+
+			log.Printf("Retryable error: %v", err)
+			errVal.Store(err)
 		}
+	}()
+
+	// Wait for completion
+	select {
+	case <-doneCh:
+	case <-ctx.Done():
 	}
+
+	// Check if we have a context error to check if we're interrupted or timeout
+	switch ctx.Err() {
+	case context.Canceled:
+		return fmt.Errorf("interrupted")
+	case context.DeadlineExceeded:
+		return fmt.Errorf("timeout")
+	}
+
+	// Check if we got an error executing
+	if err, ok := errVal.Load().(error); ok {
+		return err
+	}
+
+	return nil
 }

terraform/context.go

@@ -632,10 +632,14 @@ func (c *Context) Refresh() (*State, error) {
 //
 // Stop will block until the task completes.
 func (c *Context) Stop() {
+	log.Printf("[WARN] terraform: Stop called, initiating interrupt sequence")
+
 	c.l.Lock()
 
 	// If we're running, then stop
 	if c.runContextCancel != nil {
+		log.Printf("[WARN] terraform: run context exists, stopping")
+
 		// Tell the hook we want to stop
 		c.sh.Stop()
 
@@ -652,8 +656,11 @@ func (c *Context) Stop() {
 
 	// Wait if we have a context
 	if ctx != nil {
+		log.Printf("[WARN] terraform: stop waiting for context completion")
 		<-ctx.Done()
 	}
+
+	log.Printf("[WARN] terraform: stop complete")
 }
 
 // Validate validates the configuration and returns any warnings or errors.