Merge pull request #16290 from hashicorp/jbardin/s3-errors

retry on s3 state upload errors

backend/remote-state/s3/client.go

@@ -149,6 +149,9 @@ func (c *RemoteClient) Put(data []byte) error {
 	contentType := "application/json"
 	contentLength := int64(len(data))
 
+	// we immediately retry on an internal error, as those are usually transient
+	maxRetries := 2
+	for retryCount := 0; ; retryCount++ {
 		i := &s3.PutObjectInput{
 			ContentType:   &contentType,
 			ContentLength: &contentLength,
@@ -174,7 +177,18 @@ func (c *RemoteClient) Put(data []byte) error {
 
 		_, err := c.s3Client.PutObject(i)
 		if err != nil {
-		return fmt.Errorf("Failed to upload state: %v", err)
+			if awserr, ok := err.(awserr.Error); ok {
+				if awserr.Code() == s3ErrCodeInternalError {
+					if retryCount > maxRetries {
+						return fmt.Errorf("failed to upload state: %s", err)
+					}
+					log.Println("[WARN] s3 internal error, retrying...")
+					continue
+				}
+			}
+			return fmt.Errorf("failed to upload state: %s", err)
+		}
+		break
 	}
 
 	sum := md5.Sum(data)