backend/remote: also retry on server errors

Enably retrying on server errors in the updated `go-tfe` client and add a retry log hook for writing retry messages to the CLI.

backend/remote/backend.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"time"
 
 	tfe "github.com/hashicorp/go-tfe"
 	version "github.com/hashicorp/go-version"
@@ -56,6 +57,9 @@ type Remote struct {
 	// client is the remote backend API client.
 	client *tfe.Client
 
+	// lastRetry is set to the last time a request was retried.
+	lastRetry time.Time
+
 	// hostname of the remote backend server.
 	hostname string
 
@@ -279,10 +283,11 @@ func (b *Remote) Configure(obj cty.Value) tfdiags.Diagnostics {
 	}
 
 	cfg := &tfe.Config{
-		Address:  service.String(),
-		BasePath: service.Path,
-		Token:    token,
-		Headers:  make(http.Header),
+		Address:      service.String(),
+		BasePath:     service.Path,
+		Token:        token,
+		Headers:      make(http.Header),
+		RetryLogHook: b.retryLogHook,
 	}
 
 	// Set the version header to the current version.
@@ -324,6 +329,9 @@ func (b *Remote) Configure(obj cty.Value) tfdiags.Diagnostics {
 	b.local = backendLocal.NewWithBackend(b)
 	b.forceLocal = b.forceLocal || !entitlements.Operations
 
+	// Enable retries for server errors as the backend is now fully configured.
+	b.client.RetryServerErrors(true)
+
 	return diags
 }
 
@@ -470,6 +478,31 @@ func (b *Remote) token() (string, error) {
 	return "", nil
 }
 
+// retryLogHook is invoked each time a request is retried allowing the
+// backend to log any connection issues to prevent data loss.
+func (b *Remote) retryLogHook(attemptNum int, resp *http.Response) {
+	if b.CLI != nil {
+		// Ignore the first retry to make sure any delayed output will
+		// be written to the console before we start logging retries.
+		//
+		// The retry logic in the TFE client will retry both rate limited
+		// requests and server errors, but in the remote backend we only
+		// care about server errors so we ignore rate limit (429) errors.
+		if attemptNum == 0 || resp.StatusCode == 429 {
+			// Reset the last retry time.
+			b.lastRetry = time.Now()
+			return
+		}
+
+		if attemptNum == 1 {
+			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(initialRetryError)))
+		} else {
+			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(
+				fmt.Sprintf(repeatedRetryError, time.Since(b.lastRetry).Round(time.Second)))))
+		}
+	}
+}
+
 // Workspaces implements backend.Enhanced.
 func (b *Remote) Workspaces() ([]string, error) {
 	if b.prefix == "" {
@@ -858,6 +891,17 @@ func checkConstraintsWarning(err error) tfdiags.Diagnostic {
 	)
 }
 
+// The newline in this error is to make it look good in the CLI!
+const initialRetryError = `
+[reset][yellow]There was an error connecting to the remote backend. Please do not exit
+Terraform to prevent data loss! Trying to restore the connection...
+[reset]
+`
+
+const repeatedRetryError = `
+[reset][yellow]Still trying to restore the connection... (%s elapsed)[reset]
+`
+
 const operationCanceled = `
 [reset][red]The remote operation was successfully cancelled.[reset]
 `