Merge pull request #4407 from hashicorp/phinze/kinesis-tests

provider/aws: kinesis firehose stream: retry through IAM propagation
This commit is contained in:
Paul Hinze 2015-12-21 15:03:42 -06:00
commit 3c5524ace5
2 changed files with 200 additions and 14 deletions

View File

@ -2,6 +2,7 @@ package aws
import ( import (
"fmt" "fmt"
"log"
"strings" "strings"
"time" "time"
@ -102,7 +103,7 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta
DeliveryStreamName: aws.String(sn), DeliveryStreamName: aws.String(sn),
} }
s3_config := &firehose.S3DestinationConfiguration{ s3Config := &firehose.S3DestinationConfiguration{
BucketARN: aws.String(d.Get("s3_bucket_arn").(string)), BucketARN: aws.String(d.Get("s3_bucket_arn").(string)),
RoleARN: aws.String(d.Get("role_arn").(string)), RoleARN: aws.String(d.Get("role_arn").(string)),
BufferingHints: &firehose.BufferingHints{ BufferingHints: &firehose.BufferingHints{
@ -112,12 +113,25 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta
CompressionFormat: aws.String(d.Get("s3_data_compression").(string)), CompressionFormat: aws.String(d.Get("s3_data_compression").(string)),
} }
if v, ok := d.GetOk("s3_prefix"); ok { if v, ok := d.GetOk("s3_prefix"); ok {
s3_config.Prefix = aws.String(v.(string)) s3Config.Prefix = aws.String(v.(string))
} }
input.S3DestinationConfiguration = s3_config input.S3DestinationConfiguration = s3Config
var err error
for i := 0; i < 5; i++ {
_, err := conn.CreateDeliveryStream(input) _, err := conn.CreateDeliveryStream(input)
if awsErr, ok := err.(awserr.Error); ok {
// IAM roles can take ~10 seconds to propagate in AWS:
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
if awsErr.Code() == "InvalidArgumentException" && strings.Contains(awsErr.Message(), "Firehose is unable to assume role") {
log.Printf("[DEBUG] Firehose could not assume role referenced, retrying...")
time.Sleep(2 * time.Second)
continue
}
}
break
}
if err != nil { if err != nil {
if awsErr, ok := err.(awserr.Error); ok { if awsErr, ok := err.(awserr.Error); ok {
return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code())

View File

@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"log" "log"
"math/rand" "math/rand"
"os"
"strings" "strings"
"testing" "testing"
"time" "time"
@ -16,12 +17,17 @@ import (
func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) { func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) {
var stream firehose.DeliveryStreamDescription var stream firehose.DeliveryStreamDescription
ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int() ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_basic, ri, ri) config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_basic,
os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
resource.Test(t, resource.TestCase{ resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) }, PreCheck: func() {
testAccPreCheck(t)
if os.Getenv("AWS_ACCOUNT_ID") == "" {
t.Fatal("AWS_ACCOUNT_ID must be set")
}
},
Providers: testAccProviders, Providers: testAccProviders,
CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy,
Steps: []resource.TestStep{ Steps: []resource.TestStep{
@ -40,11 +46,18 @@ func TestAccAWSKinesisFirehoseDeliveryStream_s3ConfigUpdates(t *testing.T) {
var stream firehose.DeliveryStreamDescription var stream firehose.DeliveryStreamDescription
ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int() ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
preconfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3, ri, ri) preconfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3,
postConfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3Updates, ri, ri) os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
postConfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3Updates,
os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
resource.Test(t, resource.TestCase{ resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) }, PreCheck: func() {
testAccPreCheck(t)
if os.Getenv("AWS_ACCOUNT_ID") == "" {
t.Fatal("AWS_ACCOUNT_ID must be set")
}
},
Providers: testAccProviders, Providers: testAccProviders,
CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy,
Steps: []resource.TestStep{ Steps: []resource.TestStep{
@ -147,41 +160,200 @@ func testAccCheckKinesisFirehoseDeliveryStreamDestroy(s *terraform.State) error
} }
var testAccKinesisFirehoseDeliveryStreamConfig_basic = ` var testAccKinesisFirehoseDeliveryStreamConfig_basic = `
resource "aws_iam_role" "firehose" {
name = "terraform_acctest_firehose_delivery_role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": "%s"
}
}
}
]
}
EOF
}
resource "aws_s3_bucket" "bucket" { resource "aws_s3_bucket" "bucket" {
bucket = "tf-test-bucket-%d" bucket = "tf-test-bucket-%d"
acl = "private" acl = "private"
} }
resource "aws_iam_role_policy" "firehose" {
name = "terraform_acctest_firehose_delivery_policy"
role = "${aws_iam_role.firehose.id}"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::${aws_s3_bucket.bucket.id}",
"arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
]
}
]
}
EOF
}
resource "aws_kinesis_firehose_delivery_stream" "test_stream" { resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
depends_on = ["aws_iam_role_policy.firehose"]
name = "terraform-kinesis-firehose-basictest-%d" name = "terraform-kinesis-firehose-basictest-%d"
destination = "s3" destination = "s3"
role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" role_arn = "${aws_iam_role.firehose.arn}"
s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
}` }`
var testAccKinesisFirehoseDeliveryStreamConfig_s3 = ` var testAccKinesisFirehoseDeliveryStreamConfig_s3 = `
resource "aws_iam_role" "firehose" {
name = "terraform_acctest_firehose_delivery_role_s3"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": "%s"
}
}
}
]
}
EOF
}
resource "aws_s3_bucket" "bucket" { resource "aws_s3_bucket" "bucket" {
bucket = "tf-test-bucket-%d" bucket = "tf-test-bucket-%d"
acl = "private" acl = "private"
} }
resource "aws_iam_role_policy" "firehose" {
name = "terraform_acctest_firehose_delivery_policy_s3"
role = "${aws_iam_role.firehose.id}"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::${aws_s3_bucket.bucket.id}",
"arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
]
}
]
}
EOF
}
resource "aws_kinesis_firehose_delivery_stream" "test_stream" { resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
depends_on = ["aws_iam_role_policy.firehose"]
name = "terraform-kinesis-firehose-s3test-%d" name = "terraform-kinesis-firehose-s3test-%d"
destination = "s3" destination = "s3"
role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" role_arn = "${aws_iam_role.firehose.arn}"
s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
}` }`
var testAccKinesisFirehoseDeliveryStreamConfig_s3Updates = ` var testAccKinesisFirehoseDeliveryStreamConfig_s3Updates = `
resource "aws_iam_role" "firehose" {
name = "terraform_acctest_firehose_delivery_role_s3"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "firehose.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": "%s"
}
}
}
]
}
EOF
}
resource "aws_s3_bucket" "bucket" { resource "aws_s3_bucket" "bucket" {
bucket = "tf-test-bucket-01-%d" bucket = "tf-test-bucket-%d"
acl = "private" acl = "private"
} }
resource "aws_iam_role_policy" "firehose" {
name = "terraform_acctest_firehose_delivery_policy_s3"
role = "${aws_iam_role.firehose.id}"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::${aws_s3_bucket.bucket.id}",
"arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
]
}
]
}
EOF
}
resource "aws_kinesis_firehose_delivery_stream" "test_stream" { resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
depends_on = ["aws_iam_role_policy.firehose"]
name = "terraform-kinesis-firehose-s3test-%d" name = "terraform-kinesis-firehose-s3test-%d"
destination = "s3" destination = "s3"
role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" role_arn = "${aws_iam_role.firehose.arn}"
s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
s3_buffer_size = 10 s3_buffer_size = 10
s3_buffer_interval = 400 s3_buffer_interval = 400