ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #16243 from hashicorp/jbardin/s3-errors 

retry on s3 backend internal errors
This commit is contained in:
James Bardin 2017-10-04 09:26:26 -04:00 committed by GitHub
parent 00c3367dda fba02f0bea
commit b20ab9f0bb
1 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
backend/remote-state/s3/client.go
View File

@ -22,7 +22,10 @@ import (
) )
// Store the last saved serial in dynamo with this suffix for consistency checks. // Store the last saved serial in dynamo with this suffix for consistency checks.
const stateIDSuffix = "-md5" const (
stateIDSuffix = "-md5"
s3ErrCodeInternalError = "InternalError"
)
type RemoteClient struct { type RemoteClient struct {
s3Client *s3.S3 s3Client *s3.S3
@ -92,21 +95,33 @@ func (c *RemoteClient) Get() (payload *remote.Payload, err error) {
} }
func (c *RemoteClient) get() (*remote.Payload, error) { func (c *RemoteClient) get() (*remote.Payload, error) {
output, err := c.s3Client.GetObject(&s3.GetObjectInput{ var output *s3.GetObjectOutput
Bucket: &c.bucketName, var err error
Key: &c.path,
})
if err != nil { // we immediately retry on an internal error, as those are usually transient
if awserr := err.(awserr.Error); awserr != nil { maxRetries := 2
if awserr.Code() == "NoSuchKey" { for retryCount := 0; ; retryCount++ {
return nil, nil output, err = c.s3Client.GetObject(&s3.GetObjectInput{
} else { Bucket: &c.bucketName,
return nil, err Key: &c.path,
})
if err != nil {
if awserr, ok := err.(awserr.Error); ok {
switch awserr.Code() {
case s3.ErrCodeNoSuchKey:
return nil, nil
case s3ErrCodeInternalError:
if retryCount > maxRetries {
return nil, err
}
log.Println("[WARN] s3 internal error, retrying...")
continue
}
} }
} else {
return nil, err return nil, err
} }
break
} }
defer output.Body.Close() defer output.Body.Close()