ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

retry on s3 state upload errors 

While #16243 added the ability to retry getting a state from S3, Put can
return the same InternalError status. Use the same retry logic when
uploading state to S3.
This commit is contained in:
James Bardin 2017-10-08 16:46:33 -04:00
parent 7916268d7f
commit 611b1ced59
1 changed files with 36 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
backend/remote-state/s3/client.go
View File

@ -149,32 +149,46 @@ func (c *RemoteClient) Put(data []byte) error {
contentType := "application/json" contentType := "application/json"
contentLength := int64(len(data)) contentLength := int64(len(data))
i := &s3.PutObjectInput{ // we immediately retry on an internal error, as those are usually transient
ContentType: &contentType, maxRetries := 2
ContentLength: &contentLength, for retryCount := 0; ; retryCount++ {
Body: bytes.NewReader(data), i := &s3.PutObjectInput{
Bucket: &c.bucketName, ContentType: &contentType,
Key: &c.path, ContentLength: &contentLength,
} Body: bytes.NewReader(data),
Bucket: &c.bucketName,
if c.serverSideEncryption { Key: &c.path,
if c.kmsKeyID != "" {
i.SSEKMSKeyId = &c.kmsKeyID
i.ServerSideEncryption = aws.String("aws:kms")
} else {
i.ServerSideEncryption = aws.String("AES256")
} }
}
if c.acl != "" { if c.serverSideEncryption {
i.ACL = aws.String(c.acl) if c.kmsKeyID != "" {
} i.SSEKMSKeyId = &c.kmsKeyID
i.ServerSideEncryption = aws.String("aws:kms")
} else {
i.ServerSideEncryption = aws.String("AES256")
}
}
log.Printf("[DEBUG] Uploading remote state to S3: %#v", i) if c.acl != "" {
i.ACL = aws.String(c.acl)
}
_, err := c.s3Client.PutObject(i) log.Printf("[DEBUG] Uploading remote state to S3: %#v", i)
if err != nil {
return fmt.Errorf("Failed to upload state: %v", err) _, err := c.s3Client.PutObject(i)
if err != nil {
if awserr, ok := err.(awserr.Error); ok {
if awserr.Code() == s3ErrCodeInternalError {
if retryCount > maxRetries {
return fmt.Errorf("failed to upload state: %s", err)
}
log.Println("[WARN] s3 internal error, retrying...")
continue
}
}
return fmt.Errorf("failed to upload state: %s", err)
}
break
} }
sum := md5.Sum(data) sum := md5.Sum(data)