provider/aws: kinesis firehose stream: retry through IAM propagation

As I was fixing up the AccTests to not depend on a single existing IAM
role (which this commit does), I noticed that without some sleeping that
the kinesis_firehose_delivery_stream would often come back with:

```
msg: Firehose is unable to assume role {{arn}}. Please check the role provided.
code: InvalidArgumentException
```

Similar to the strategy taken in aws_instance with IAM Instance Profile errors,
I dropped in a simple retry loop which seemed to take care of the issue. Seems
that the same permission propagation delays apply here too.

builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go

@@ -2,6 +2,7 @@ package aws
 
 import (
 	"fmt"
+	"log"
 	"strings"
 	"time"
 
@@ -102,7 +103,7 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta
 		DeliveryStreamName: aws.String(sn),
 	}
 
-	s3_config := &firehose.S3DestinationConfiguration{
+	s3Config := &firehose.S3DestinationConfiguration{
 		BucketARN: aws.String(d.Get("s3_bucket_arn").(string)),
 		RoleARN:   aws.String(d.Get("role_arn").(string)),
 		BufferingHints: &firehose.BufferingHints{
@@ -112,12 +113,25 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta
 		CompressionFormat: aws.String(d.Get("s3_data_compression").(string)),
 	}
 	if v, ok := d.GetOk("s3_prefix"); ok {
-		s3_config.Prefix = aws.String(v.(string))
+		s3Config.Prefix = aws.String(v.(string))
 	}
 
-	input.S3DestinationConfiguration = s3_config
+	input.S3DestinationConfiguration = s3Config
 
+	var err error
+	for i := 0; i < 5; i++ {
 		_, err := conn.CreateDeliveryStream(input)
+		if awsErr, ok := err.(awserr.Error); ok {
+			// IAM roles can take ~10 seconds to propagate in AWS:
+			//  http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console
+			if awsErr.Code() == "InvalidArgumentException" && strings.Contains(awsErr.Message(), "Firehose is unable to assume role") {
+				log.Printf("[DEBUG] Firehose could not assume role referenced, retrying...")
+				time.Sleep(2 * time.Second)
+				continue
+			}
+		}
+		break
+	}
 	if err != nil {
 		if awsErr, ok := err.(awserr.Error); ok {
 			return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code())

builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"math/rand"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -16,12 +17,17 @@ import (
 
 func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) {
 	var stream firehose.DeliveryStreamDescription
-
 	ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
-	config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_basic, ri, ri)
+	config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_basic,
+		os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
 
 	resource.Test(t, resource.TestCase{
-		PreCheck:     func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			if os.Getenv("AWS_ACCOUNT_ID") == "" {
+				t.Fatal("AWS_ACCOUNT_ID must be set")
+			}
+		},
 		Providers:    testAccProviders,
 		CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy,
 		Steps: []resource.TestStep{
@@ -40,11 +46,18 @@ func TestAccAWSKinesisFirehoseDeliveryStream_s3ConfigUpdates(t *testing.T) {
 	var stream firehose.DeliveryStreamDescription
 
 	ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int()
-	preconfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3, ri, ri)
+	preconfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3,
-	postConfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3Updates, ri, ri)
+		os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
+	postConfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3Updates,
+		os.Getenv("AWS_ACCOUNT_ID"), ri, ri)
 
 	resource.Test(t, resource.TestCase{
-		PreCheck:     func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			if os.Getenv("AWS_ACCOUNT_ID") == "" {
+				t.Fatal("AWS_ACCOUNT_ID must be set")
+			}
+		},
 		Providers:    testAccProviders,
 		CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy,
 		Steps: []resource.TestStep{
@@ -147,41 +160,200 @@ func testAccCheckKinesisFirehoseDeliveryStreamDestroy(s *terraform.State) error
 }
 
 var testAccKinesisFirehoseDeliveryStreamConfig_basic = `
+resource "aws_iam_role" "firehose" {
+	name = "terraform_acctest_firehose_delivery_role"
+	assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "firehose.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole",
+      "Condition": {
+        "StringEquals": {
+          "sts:ExternalId": "%s"
+        }
+      }
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_s3_bucket" "bucket" {
 	bucket = "tf-test-bucket-%d"
 	acl = "private"
 }
 
+resource "aws_iam_role_policy" "firehose" {
+	name = "terraform_acctest_firehose_delivery_policy"
+	role = "${aws_iam_role.firehose.id}"
+	policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}",
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
+      ]
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
+	depends_on = ["aws_iam_role_policy.firehose"]
 	name = "terraform-kinesis-firehose-basictest-%d"
 	destination = "s3"
-	role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role"
+	role_arn = "${aws_iam_role.firehose.arn}"
 	s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
 }`
 
 var testAccKinesisFirehoseDeliveryStreamConfig_s3 = `
+resource "aws_iam_role" "firehose" {
+	name = "terraform_acctest_firehose_delivery_role_s3"
+	assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "firehose.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole",
+      "Condition": {
+        "StringEquals": {
+          "sts:ExternalId": "%s"
+        }
+      }
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_s3_bucket" "bucket" {
 	bucket = "tf-test-bucket-%d"
 	acl = "private"
 }
 
+resource "aws_iam_role_policy" "firehose" {
+	name = "terraform_acctest_firehose_delivery_policy_s3"
+	role = "${aws_iam_role.firehose.id}"
+	policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}",
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
+      ]
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
+	depends_on = ["aws_iam_role_policy.firehose"]
 	name = "terraform-kinesis-firehose-s3test-%d"
 	destination = "s3"
-	role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role"
+	role_arn = "${aws_iam_role.firehose.arn}"
 	s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
 }`
 
 var testAccKinesisFirehoseDeliveryStreamConfig_s3Updates = `
+resource "aws_iam_role" "firehose" {
+	name = "terraform_acctest_firehose_delivery_role_s3"
+	assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Principal": {
+        "Service": "firehose.amazonaws.com"
+      },
+      "Action": "sts:AssumeRole",
+      "Condition": {
+        "StringEquals": {
+          "sts:ExternalId": "%s"
+        }
+      }
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_s3_bucket" "bucket" {
-	bucket = "tf-test-bucket-01-%d"
+	bucket = "tf-test-bucket-%d"
 	acl = "private"
 }
 
+resource "aws_iam_role_policy" "firehose" {
+	name = "terraform_acctest_firehose_delivery_policy_s3"
+	role = "${aws_iam_role.firehose.id}"
+	policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "",
+      "Effect": "Allow",
+      "Action": [
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation",
+        "s3:GetObject",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}",
+        "arn:aws:s3:::${aws_s3_bucket.bucket.id}/*"
+      ]
+    }
+  ]
+}
+EOF
+}
+
 resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
+	depends_on = ["aws_iam_role_policy.firehose"]
 	name = "terraform-kinesis-firehose-s3test-%d"
 	destination = "s3"
-	role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role"
+	role_arn = "${aws_iam_role.firehose.arn}"
 	s3_bucket_arn = "${aws_s3_bucket.bucket.arn}"
 	s3_buffer_size = 10
 	s3_buffer_interval = 400