From fc983c5505c8f332f7f199afa6fb48222c88f18a Mon Sep 17 00:00:00 2001 From: stack72 Date: Mon, 9 Nov 2015 22:26:55 +0000 Subject: [PATCH 1/6] Initial Create, Read and Delete work for the S3 part of the Kinesis Firehose resource --- builtin/providers/aws/config.go | 5 + builtin/providers/aws/provider.go | 203 ++++++++--------- .../resource_aws_kinesis_firehose_stream.go | 210 ++++++++++++++++++ 3 files changed, 317 insertions(+), 101 deletions(-) create mode 100644 builtin/providers/aws/resource_aws_kinesis_firehose_stream.go diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index 3e835c106..bd7b08924 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -27,6 +27,7 @@ import ( "github.com/aws/aws-sdk-go/service/elasticache" elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/firehose" "github.com/aws/aws-sdk-go/service/glacier" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/kinesis" @@ -74,6 +75,7 @@ type AWSClient struct { rdsconn *rds.RDS iamconn *iam.IAM kinesisconn *kinesis.Kinesis + firehoseconn *firehose.Firehose elasticacheconn *elasticache.ElastiCache lambdaconn *lambda.Lambda opsworksconn *opsworks.OpsWorks @@ -168,6 +170,9 @@ func (c *Config) Client() (interface{}, error) { errs = append(errs, authErr) } + log.Println("[INFO] Initializing Kinesis Firehose Connection") + client.firehoseconn = autoscaling.New(sess) + log.Println("[INFO] Initializing AutoScaling connection") client.autoscalingconn = autoscaling.New(sess) diff --git a/builtin/providers/aws/provider.go b/builtin/providers/aws/provider.go index b5392429a..dfa1565fc 100644 --- a/builtin/providers/aws/provider.go +++ b/builtin/providers/aws/provider.go @@ -163,107 +163,108 @@ func Provider() terraform.ResourceProvider { }, ResourcesMap: map[string]*schema.Resource{ - "aws_ami": resourceAwsAmi(), - "aws_ami_copy": resourceAwsAmiCopy(), - "aws_ami_from_instance": resourceAwsAmiFromInstance(), - "aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(), - "aws_autoscaling_group": resourceAwsAutoscalingGroup(), - "aws_autoscaling_notification": resourceAwsAutoscalingNotification(), - "aws_autoscaling_policy": resourceAwsAutoscalingPolicy(), - "aws_cloudformation_stack": resourceAwsCloudFormationStack(), - "aws_cloudtrail": resourceAwsCloudTrail(), - "aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(), - "aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(), - "aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(), - "aws_codedeploy_app": resourceAwsCodeDeployApp(), - "aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(), - "aws_codecommit_repository": resourceAwsCodeCommitRepository(), - "aws_customer_gateway": resourceAwsCustomerGateway(), - "aws_db_instance": resourceAwsDbInstance(), - "aws_db_parameter_group": resourceAwsDbParameterGroup(), - "aws_db_security_group": resourceAwsDbSecurityGroup(), - "aws_db_subnet_group": resourceAwsDbSubnetGroup(), - "aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(), - "aws_dynamodb_table": resourceAwsDynamoDbTable(), - "aws_ebs_volume": resourceAwsEbsVolume(), - "aws_ecs_cluster": resourceAwsEcsCluster(), - "aws_ecs_service": resourceAwsEcsService(), - "aws_ecs_task_definition": resourceAwsEcsTaskDefinition(), - "aws_efs_file_system": resourceAwsEfsFileSystem(), - "aws_efs_mount_target": resourceAwsEfsMountTarget(), - "aws_eip": resourceAwsEip(), - "aws_elasticache_cluster": resourceAwsElasticacheCluster(), - "aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(), - "aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(), - "aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(), - "aws_elasticsearch_domain": resourceAwsElasticSearchDomain(), - "aws_elb": resourceAwsElb(), - "aws_flow_log": resourceAwsFlowLog(), - "aws_glacier_vault": resourceAwsGlacierVault(), - "aws_iam_access_key": resourceAwsIamAccessKey(), - "aws_iam_group_policy": resourceAwsIamGroupPolicy(), - "aws_iam_group": resourceAwsIamGroup(), - "aws_iam_group_membership": resourceAwsIamGroupMembership(), - "aws_iam_instance_profile": resourceAwsIamInstanceProfile(), - "aws_iam_policy": resourceAwsIamPolicy(), - "aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(), - "aws_iam_role_policy": resourceAwsIamRolePolicy(), - "aws_iam_role": resourceAwsIamRole(), - "aws_iam_saml_provider": resourceAwsIamSamlProvider(), - "aws_iam_server_certificate": resourceAwsIAMServerCertificate(), - "aws_iam_user_policy": resourceAwsIamUserPolicy(), - "aws_iam_user": resourceAwsIamUser(), - "aws_instance": resourceAwsInstance(), - "aws_internet_gateway": resourceAwsInternetGateway(), - "aws_key_pair": resourceAwsKeyPair(), - "aws_kinesis_stream": resourceAwsKinesisStream(), - "aws_lambda_function": resourceAwsLambdaFunction(), - "aws_launch_configuration": resourceAwsLaunchConfiguration(), - "aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(), - "aws_main_route_table_association": resourceAwsMainRouteTableAssociation(), - "aws_network_acl": resourceAwsNetworkAcl(), - "aws_network_interface": resourceAwsNetworkInterface(), - "aws_opsworks_stack": resourceAwsOpsworksStack(), - "aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(), - "aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(), - "aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(), - "aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(), - "aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(), - "aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(), - "aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(), - "aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(), - "aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(), - "aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(), - "aws_placement_group": resourceAwsPlacementGroup(), - "aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(), - "aws_rds_cluster": resourceAwsRDSCluster(), - "aws_rds_cluster_instance": resourceAwsRDSClusterInstance(), - "aws_route53_delegation_set": resourceAwsRoute53DelegationSet(), - "aws_route53_record": resourceAwsRoute53Record(), - "aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(), - "aws_route53_zone": resourceAwsRoute53Zone(), - "aws_route53_health_check": resourceAwsRoute53HealthCheck(), - "aws_route": resourceAwsRoute(), - "aws_route_table": resourceAwsRouteTable(), - "aws_route_table_association": resourceAwsRouteTableAssociation(), - "aws_s3_bucket": resourceAwsS3Bucket(), - "aws_s3_bucket_object": resourceAwsS3BucketObject(), - "aws_security_group": resourceAwsSecurityGroup(), - "aws_security_group_rule": resourceAwsSecurityGroupRule(), - "aws_spot_instance_request": resourceAwsSpotInstanceRequest(), - "aws_sqs_queue": resourceAwsSqsQueue(), - "aws_sns_topic": resourceAwsSnsTopic(), - "aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(), - "aws_subnet": resourceAwsSubnet(), - "aws_volume_attachment": resourceAwsVolumeAttachment(), - "aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(), - "aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(), - "aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(), - "aws_vpc": resourceAwsVpc(), - "aws_vpc_endpoint": resourceAwsVpcEndpoint(), - "aws_vpn_connection": resourceAwsVpnConnection(), - "aws_vpn_connection_route": resourceAwsVpnConnectionRoute(), - "aws_vpn_gateway": resourceAwsVpnGateway(), + "aws_ami": resourceAwsAmi(), + "aws_ami_copy": resourceAwsAmiCopy(), + "aws_ami_from_instance": resourceAwsAmiFromInstance(), + "aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(), + "aws_autoscaling_group": resourceAwsAutoscalingGroup(), + "aws_autoscaling_notification": resourceAwsAutoscalingNotification(), + "aws_autoscaling_policy": resourceAwsAutoscalingPolicy(), + "aws_cloudformation_stack": resourceAwsCloudFormationStack(), + "aws_cloudtrail": resourceAwsCloudTrail(), + "aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(), + "aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(), + "aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(), + "aws_codedeploy_app": resourceAwsCodeDeployApp(), + "aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(), + "aws_codecommit_repository": resourceAwsCodeCommitRepository(), + "aws_customer_gateway": resourceAwsCustomerGateway(), + "aws_db_instance": resourceAwsDbInstance(), + "aws_db_parameter_group": resourceAwsDbParameterGroup(), + "aws_db_security_group": resourceAwsDbSecurityGroup(), + "aws_db_subnet_group": resourceAwsDbSubnetGroup(), + "aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(), + "aws_dynamodb_table": resourceAwsDynamoDbTable(), + "aws_ebs_volume": resourceAwsEbsVolume(), + "aws_ecs_cluster": resourceAwsEcsCluster(), + "aws_ecs_service": resourceAwsEcsService(), + "aws_ecs_task_definition": resourceAwsEcsTaskDefinition(), + "aws_efs_file_system": resourceAwsEfsFileSystem(), + "aws_efs_mount_target": resourceAwsEfsMountTarget(), + "aws_eip": resourceAwsEip(), + "aws_elasticache_cluster": resourceAwsElasticacheCluster(), + "aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(), + "aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(), + "aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(), + "aws_elasticsearch_domain": resourceAwsElasticSearchDomain(), + "aws_elb": resourceAwsElb(), + "aws_flow_log": resourceAwsFlowLog(), + "aws_glacier_vault": resourceAwsGlacierVault(), + "aws_iam_access_key": resourceAwsIamAccessKey(), + "aws_iam_group_policy": resourceAwsIamGroupPolicy(), + "aws_iam_group": resourceAwsIamGroup(), + "aws_iam_group_membership": resourceAwsIamGroupMembership(), + "aws_iam_instance_profile": resourceAwsIamInstanceProfile(), + "aws_iam_policy": resourceAwsIamPolicy(), + "aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(), + "aws_iam_role_policy": resourceAwsIamRolePolicy(), + "aws_iam_role": resourceAwsIamRole(), + "aws_iam_saml_provider": resourceAwsIamSamlProvider(), + "aws_iam_server_certificate": resourceAwsIAMServerCertificate(), + "aws_iam_user_policy": resourceAwsIamUserPolicy(), + "aws_iam_user": resourceAwsIamUser(), + "aws_instance": resourceAwsInstance(), + "aws_internet_gateway": resourceAwsInternetGateway(), + "aws_key_pair": resourceAwsKeyPair(), + "aws_kinesis_firehose_delivery_stream": resourceAwsKinesisFirehoseDeliveryStream(), + "aws_kinesis_stream": resourceAwsKinesisStream(), + "aws_lambda_function": resourceAwsLambdaFunction(), + "aws_launch_configuration": resourceAwsLaunchConfiguration(), + "aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(), + "aws_main_route_table_association": resourceAwsMainRouteTableAssociation(), + "aws_network_acl": resourceAwsNetworkAcl(), + "aws_network_interface": resourceAwsNetworkInterface(), + "aws_opsworks_stack": resourceAwsOpsworksStack(), + "aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(), + "aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(), + "aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(), + "aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(), + "aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(), + "aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(), + "aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(), + "aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(), + "aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(), + "aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(), + "aws_placement_group": resourceAwsPlacementGroup(), + "aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(), + "aws_rds_cluster": resourceAwsRDSCluster(), + "aws_rds_cluster_instance": resourceAwsRDSClusterInstance(), + "aws_route53_delegation_set": resourceAwsRoute53DelegationSet(), + "aws_route53_record": resourceAwsRoute53Record(), + "aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(), + "aws_route53_zone": resourceAwsRoute53Zone(), + "aws_route53_health_check": resourceAwsRoute53HealthCheck(), + "aws_route": resourceAwsRoute(), + "aws_route_table": resourceAwsRouteTable(), + "aws_route_table_association": resourceAwsRouteTableAssociation(), + "aws_s3_bucket": resourceAwsS3Bucket(), + "aws_s3_bucket_object": resourceAwsS3BucketObject(), + "aws_security_group": resourceAwsSecurityGroup(), + "aws_security_group_rule": resourceAwsSecurityGroupRule(), + "aws_spot_instance_request": resourceAwsSpotInstanceRequest(), + "aws_sqs_queue": resourceAwsSqsQueue(), + "aws_sns_topic": resourceAwsSnsTopic(), + "aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(), + "aws_subnet": resourceAwsSubnet(), + "aws_volume_attachment": resourceAwsVolumeAttachment(), + "aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(), + "aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(), + "aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(), + "aws_vpc": resourceAwsVpc(), + "aws_vpc_endpoint": resourceAwsVpcEndpoint(), + "aws_vpn_connection": resourceAwsVpnConnection(), + "aws_vpn_connection_route": resourceAwsVpnConnectionRoute(), + "aws_vpn_gateway": resourceAwsVpnGateway(), }, ConfigureFunc: providerConfigure, diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go b/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go new file mode 100644 index 000000000..320e078b2 --- /dev/null +++ b/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go @@ -0,0 +1,210 @@ +package aws + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/firehose" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsKinesisFirehoseDeliveryStreamCreate, + Read: resourceAwsKinesisFirehoseDeliveryStreamRead, + Update: resourceAwsKinesisFirehoseDeliveryStreamUpdate, + Delete: resourceAwsKinesisFirehoseDeliveryStreamDelete, + + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "destination": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "role_arn": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + + "s3_bucket_arn": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + + "s3_prefix": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + + "s3_buffer_size": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 5, + }, + + "s3_buffer_interval": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 300, + }, + + "s3_data_compression": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "UNCOMPRESSED", + }, + + "arn": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + }, + } +} + +func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + + sn := d.Get("name").(string) + input := &firehose.CreateDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + } + + s3_config := &firehose.S3DestinationConfiguration{ + BucketARN: aws.String(d.Get("s3_bucket_arn").(string)), + RoleARN: aws.String(d.Get("role_arn").(string)), + BufferingHints: &firehose.BufferingHints{ + IntervalInSeconds: aws.Int64(int64(d.Get("s3_buffer_interval").(int))), + SizeInMBs: aws.Int64(int64(d.Get("s3_buffer_size").(int))), + }, + CompressionFormat: aws.String(d.Get("s3_data_compression").(string)), + } + if v, ok := d.GetOk("s3_prefix"); ok { + s3_config.Prefix = aws.String(v.(string)) + } + + input.S3DestinationConfiguration = s3_config + + _, err := conn.CreateDeliveryStream(input) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) + } + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{"CREATING"}, + Target: "ACTIVE", + Refresh: firehoseStreamStateRefreshFunc(conn, sn), + Timeout: 5 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + firehoseStream, err := stateConf.WaitForState() + if err != nil { + return fmt.Errorf( + "Error waiting for Kinesis Stream (%s) to become active: %s", + sn, err) + } + + s := firehoseStream.(*firehose.DeliveryStreamDescription) + d.SetId(*s.DeliveryStreamARN) + d.Set("arn", s.DeliveryStreamARN) + + return resourceAwsKinesisStreamUpdate(d, meta) +} + +func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta interface{}) error { + // conn := meta.(*AWSClient).firehoseconn + + return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) +} + +func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + sn := d.Get("name").(string) + describeOpts := &firehose.DeliveryStreamDescription{ + DeliveryStreamName: aws.String(sn), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "ResourceNotFoundException" { + d.SetId("") + return nil + } + return fmt.Errorf("[WARN] Error reading Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) + } + return err + } + + s := resp.DeliveryStreamDescription + d.Set("arn", *s.DeliveryStreamARN) + + return nil +} + +func resourceAwsKinesisFirehoseDeliveryStreamDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + + sn := d.Get("name").(string) + _, err := conn.DeleteDeliveryStream(&firehose.DeleteDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + }) + + if err != nil { + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{"DELETING"}, + Target: "DESTROYED", + Refresh: firehoseStreamStateRefreshFunc(conn, sn), + Timeout: 5 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + _, err = stateConf.WaitForState() + if err != nil { + return fmt.Errorf( + "Error waiting for Delivery Stream (%s) to be destroyed: %s", + sn, err) + } + + d.SetId("") + return nil +} + +func firehoseStreamStateRefreshFunc(conn *firehose.Firehose, sn string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + describeOpts := &firehose.DescribeDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "ResourceNotFoundException" { + return 42, "DESTROYED", nil + } + return nil, awsErr.Code(), err + } + return nil, "failed", err + } + + return resp.DeliveryStreamDescription, *resp.DeliveryStreamDescription.DeliveryStreamStatus, nil + } +} From 5dfa9ac8234457b3b80eabe6ad7aecb8decc883b Mon Sep 17 00:00:00 2001 From: stack72 Date: Mon, 9 Nov 2015 22:44:26 +0000 Subject: [PATCH 2/6] Adding the shell for the acceptance tests for the AWS Kinesis Firehose work --- ...e_aws_kinesis_firehose_delivery_stream.go} | 0 ...s_kinesis_firehose_delivery_stream_test.go | 106 ++++++++++++++++++ 2 files changed, 106 insertions(+) rename builtin/providers/aws/{resource_aws_kinesis_firehose_stream.go => resource_aws_kinesis_firehose_delivery_stream.go} (100%) create mode 100644 builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go similarity index 100% rename from builtin/providers/aws/resource_aws_kinesis_firehose_stream.go rename to builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go new file mode 100644 index 000000000..1414f7ae6 --- /dev/null +++ b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go @@ -0,0 +1,106 @@ +package aws + +import ( + "fmt" + "math/rand" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/firehose" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) { + var stream firehose.DeliveryStreamDescription + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccKinesisFirehoseDeliveryStreamConfig, + Check: resource.ComposeTestCheckFunc( + testAccCheckKinesisFirehoseDeliveryStreamExists("aws_kinesis_firehose_delivery_stream.test_stream", &stream), + testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream), + ), + }, + }, + }) +} + +func testAccCheckKinesisFirehoseDeliveryStreamExists(n string, stream *firehose.DeliveryStreamDescription) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No Kinesis Firehose ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).firehoseconn + describeOpts := &firehose.DescribeDeliveryStreamInput{ + DeliveryStreamName: aws.String(rs.Primary.Attributes["name"]), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err != nil { + return err + } + + *stream = *resp.DeliveryStreamDescription + + return nil + } +} + +func testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(stream *firehose.DeliveryStreamDescription) resource.TestCheckFunc { + return func(s *terraform.State) error { + if !strings.HasPrefix(*stream.DeliveryStreamName, "terraform-kinesis-firehose-test") { + return fmt.Errorf("Bad Stream name: %s", *stream.DeliveryStreamName) + } + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_kinesis_firehose_delivery_stream" { + continue + } + if *stream.DeliveryStreamARN != rs.Primary.Attributes["arn"] { + return fmt.Errorf("Bad Delivery Stream ARN\n\t expected: %s\n\tgot: %s\n", rs.Primary.Attributes["arn"], *stream.DeliveryStreamARN) + } + } + return nil + } +} + +func testAccCheckKinesisFirehoseDeliveryStreamDestroy(s *terraform.State) error { + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_kinesis_firehose_delivery_stream" { + continue + } + conn := testAccProvider.Meta().(*AWSClient).firehoseconn + describeOpts := &firehose.DescribeDeliveryStreamInput{ + DeliveryStreamName: aws.String(rs.Primary.Attributes["name"]), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err == nil { + if resp.DeliveryStreamDescription != nil && *resp.DeliveryStreamDescription.DeliveryStreamStatus != "DELETING" { + return fmt.Errorf("Error: Delivery Stream still exists") + } + } + + return nil + + } + + return nil +} + +var testAccKinesisFirehoseDeliveryStreamConfig = fmt.Sprintf(` +resource "aws_kinesis_firehose_delivery_stream" "test_stream" { + name = "terraform-kinesis-firehose-test-%d" + +} +`, rand.New(rand.NewSource(time.Now().UnixNano())).Int()) From d14d891367b2e13cef77f5c986473fdf1af8ba02 Mon Sep 17 00:00:00 2001 From: stack72 Date: Tue, 10 Nov 2015 16:24:33 +0000 Subject: [PATCH 3/6] Finishing the first pass at Kinesis Firehose. I have only implemented the S3 configuration right now as Terraform doesn't include RedShift support --- builtin/providers/aws/config.go | 2 +- ...ce_aws_kinesis_firehose_delivery_stream.go | 77 ++++++++++++++- ...s_kinesis_firehose_delivery_stream_test.go | 97 +++++++++++++++++-- 3 files changed, 165 insertions(+), 11 deletions(-) diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index bd7b08924..d8a9ff862 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -171,7 +171,7 @@ func (c *Config) Client() (interface{}, error) { } log.Println("[INFO] Initializing Kinesis Firehose Connection") - client.firehoseconn = autoscaling.New(sess) + client.firehoseconn = firehose.New(sess) log.Println("[INFO] Initializing AutoScaling connection") client.autoscalingconn = autoscaling.New(sess) diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go index 320e078b2..c1b973454 100644 --- a/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go +++ b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go @@ -2,6 +2,7 @@ package aws import ( "fmt" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -29,6 +30,10 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { Type: schema.TypeString, Required: true, ForceNew: true, + StateFunc: func(v interface{}) string { + value := v.(string) + return strings.ToLower(value) + }, }, "role_arn": &schema.Schema{ @@ -69,6 +74,18 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { Optional: true, Computed: true, }, + + "version_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + + "destination_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + }, }, } } @@ -76,6 +93,10 @@ func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).firehoseconn + if d.Get("destination").(string) != "s3" { + return fmt.Errorf("[ERROR] AWS Kinesis Firehose only supports S3 destinations for the first implementation") + } + sn := d.Get("name").(string) input := &firehose.CreateDeliveryStreamInput{ DeliveryStreamName: aws.String(sn), @@ -124,11 +145,56 @@ func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta d.SetId(*s.DeliveryStreamARN) d.Set("arn", s.DeliveryStreamARN) - return resourceAwsKinesisStreamUpdate(d, meta) + return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) } func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta interface{}) error { - // conn := meta.(*AWSClient).firehoseconn + conn := meta.(*AWSClient).firehoseconn + + if d.Get("destination").(string) != "s3" { + return fmt.Errorf("[ERROR] AWS Kinesis Firehose only supports S3 destinations for the first implementation") + } + + sn := d.Get("name").(string) + s3_config := &firehose.S3DestinationUpdate{} + + if d.HasChange("role_arn") { + s3_config.RoleARN = aws.String(d.Get("role_arn").(string)) + } + + if d.HasChange("s3_bucket_arn") { + s3_config.BucketARN = aws.String(d.Get("s3_bucket_arn").(string)) + } + + if d.HasChange("s3_prefix") { + s3_config.Prefix = aws.String(d.Get("s3_prefix").(string)) + } + + if d.HasChange("s3_data_compression") { + s3_config.CompressionFormat = aws.String(d.Get("s3_data_compression").(string)) + } + + if d.HasChange("s3_buffer_interval") || d.HasChange("s3_buffer_size") { + bufferingHints := &firehose.BufferingHints{ + IntervalInSeconds: aws.Int64(int64(d.Get("s3_buffer_interval").(int))), + SizeInMBs: aws.Int64(int64(d.Get("s3_buffer_size").(int))), + } + s3_config.BufferingHints = bufferingHints + } + + destOpts := &firehose.UpdateDestinationInput{ + DeliveryStreamName: aws.String(sn), + CurrentDeliveryStreamVersionId: aws.String(d.Get("version_id").(string)), + DestinationId: aws.String(d.Get("destination_id").(string)), + S3DestinationUpdate: s3_config, + } + + _, err := conn.UpdateDestination(destOpts) + if err != nil { + return fmt.Errorf( + "Error Updating Kinesis Firehose Delivery Stream: %s", + sn, err) + } return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) } @@ -136,7 +202,7 @@ func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).firehoseconn sn := d.Get("name").(string) - describeOpts := &firehose.DeliveryStreamDescription{ + describeOpts := &firehose.DescribeDeliveryStreamInput{ DeliveryStreamName: aws.String(sn), } resp, err := conn.DescribeDeliveryStream(describeOpts) @@ -152,7 +218,12 @@ func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta i } s := resp.DeliveryStreamDescription + d.Set("version_id", s.VersionId) d.Set("arn", *s.DeliveryStreamARN) + if len(s.Destinations) > 0 { + destination := s.Destinations[0] + d.Set("destination_id", *destination.DestinationId) + } return nil } diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go index 1414f7ae6..611e196ce 100644 --- a/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go +++ b/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream_test.go @@ -2,6 +2,7 @@ package aws import ( "fmt" + "log" "math/rand" "strings" "testing" @@ -16,13 +17,16 @@ import ( func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) { var stream firehose.DeliveryStreamDescription + ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int() + config := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_basic, ri, ri) + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, Steps: []resource.TestStep{ resource.TestStep{ - Config: testAccKinesisFirehoseDeliveryStreamConfig, + Config: config, Check: resource.ComposeTestCheckFunc( testAccCheckKinesisFirehoseDeliveryStreamExists("aws_kinesis_firehose_delivery_stream.test_stream", &stream), testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream), @@ -32,9 +36,53 @@ func TestAccAWSKinesisFirehoseDeliveryStream_basic(t *testing.T) { }) } +func TestAccAWSKinesisFirehoseDeliveryStream_s3ConfigUpdates(t *testing.T) { + var stream firehose.DeliveryStreamDescription + + ri := rand.New(rand.NewSource(time.Now().UnixNano())).Int() + preconfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3, ri, ri) + postConfig := fmt.Sprintf(testAccKinesisFirehoseDeliveryStreamConfig_s3Updates, ri, ri) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckKinesisFirehoseDeliveryStreamDestroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: preconfig, + Check: resource.ComposeTestCheckFunc( + testAccCheckKinesisFirehoseDeliveryStreamExists("aws_kinesis_firehose_delivery_stream.test_stream", &stream), + testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_buffer_size", "5"), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_buffer_interval", "300"), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_data_compression", "UNCOMPRESSED"), + ), + }, + + resource.TestStep{ + Config: postConfig, + Check: resource.ComposeTestCheckFunc( + testAccCheckKinesisFirehoseDeliveryStreamExists("aws_kinesis_firehose_delivery_stream.test_stream", &stream), + testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(&stream), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_buffer_size", "10"), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_buffer_interval", "400"), + resource.TestCheckResourceAttr( + "aws_kinesis_firehose_delivery_stream.test_stream", "s3_data_compression", "GZIP"), + ), + }, + }, + }) +} + func testAccCheckKinesisFirehoseDeliveryStreamExists(n string, stream *firehose.DeliveryStreamDescription) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[n] + log.Printf("State: %#v", s.RootModule().Resources) if !ok { return fmt.Errorf("Not found: %s", n) } @@ -60,7 +108,7 @@ func testAccCheckKinesisFirehoseDeliveryStreamExists(n string, stream *firehose. func testAccCheckAWSKinesisFirehoseDeliveryStreamAttributes(stream *firehose.DeliveryStreamDescription) resource.TestCheckFunc { return func(s *terraform.State) error { - if !strings.HasPrefix(*stream.DeliveryStreamName, "terraform-kinesis-firehose-test") { + if !strings.HasPrefix(*stream.DeliveryStreamName, "terraform-kinesis-firehose") { return fmt.Errorf("Bad Stream name: %s", *stream.DeliveryStreamName) } for _, rs := range s.RootModule().Resources { @@ -98,9 +146,44 @@ func testAccCheckKinesisFirehoseDeliveryStreamDestroy(s *terraform.State) error return nil } -var testAccKinesisFirehoseDeliveryStreamConfig = fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test_stream" { - name = "terraform-kinesis-firehose-test-%d" - +var testAccKinesisFirehoseDeliveryStreamConfig_basic = ` +resource "aws_s3_bucket" "bucket" { + bucket = "tf-test-bucket-%d" + acl = "private" } -`, rand.New(rand.NewSource(time.Now().UnixNano())).Int()) + +resource "aws_kinesis_firehose_delivery_stream" "test_stream" { + name = "terraform-kinesis-firehose-basictest-%d" + destination = "s3" + role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" + s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" +}` + +var testAccKinesisFirehoseDeliveryStreamConfig_s3 = ` +resource "aws_s3_bucket" "bucket" { + bucket = "tf-test-bucket-%d" + acl = "private" +} + +resource "aws_kinesis_firehose_delivery_stream" "test_stream" { + name = "terraform-kinesis-firehose-s3test-%d" + destination = "s3" + role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" + s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" +}` + +var testAccKinesisFirehoseDeliveryStreamConfig_s3Updates = ` +resource "aws_s3_bucket" "bucket" { + bucket = "tf-test-bucket-01-%d" + acl = "private" +} + +resource "aws_kinesis_firehose_delivery_stream" "test_stream" { + name = "terraform-kinesis-firehose-s3test-%d" + destination = "s3" + role_arn = "arn:aws:iam::946579370547:role/firehose_delivery_role" + s3_bucket_arn = "${aws_s3_bucket.bucket.arn}" + s3_buffer_size = 10 + s3_buffer_interval = 400 + s3_data_compression = "GZIP" +}` From 8c68cb108021db980eff26a9a83956e34464470e Mon Sep 17 00:00:00 2001 From: stack72 Date: Tue, 10 Nov 2015 16:43:34 +0000 Subject: [PATCH 4/6] Adding the documentation for the Kinesis Firehose Delivery Stream --- ...sis_firehose_delivery_stream.html.markdown | 70 +++++++++++++++++++ website/source/layouts/aws.erb | 11 +++ 2 files changed, 81 insertions(+) create mode 100644 website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown diff --git a/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown new file mode 100644 index 000000000..571798385 --- /dev/null +++ b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown @@ -0,0 +1,70 @@ +--- +layout: "aws" +page_title: "AWS: aws_kinesis_firehose_delivery_stream" +sidebar_current: "docs-aws-resource-kinesis-firehose-delivery-stream" +description: |- + Provides a AWS Kinesis Firehose Delivery Stream +--- + +# aws\_kinesis\_stream + +Provides a Kinesis Firehose Delivery Stream resource. Amazon Kinesis Firehose is a fully managed, elastic service to easily deliver real-time data streams to destinations such as Amazon S3 and Amazon Redshift. + +For more details, see the [Amazon Kinesis Firehose Documentation][1]. + +## Example Usage + +``` +resource "aws_s3_bucket" "bucket" { + bucket = "tf-test-bucket" + acl = "private" +} + +esource "aws_iam_role" "firehose_role" { + name = "firehose_test_role" + assume_role_policy = < + > + Kinesis Firehose Resources + + + > Lambda Resources From 255486857e1c1574475f70df0f42802591771183 Mon Sep 17 00:00:00 2001 From: stack72 Date: Tue, 10 Nov 2015 16:45:46 +0000 Subject: [PATCH 5/6] Changing the Firehose documentation to note the regions is is available for and also that s3 is the only destination involved right now --- .../aws/r/kinesis_firehose_delivery_stream.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown index 571798385..1eec58951 100644 --- a/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown @@ -47,6 +47,8 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { } ``` +~> **NOTE:** Kinesis Firehose is currently only support in us-east-1, us-west-2 and eu-west-1. This implementation of Kinesis Firehose only supports the s3 destination type + ## Argument Reference The following arguments are supported: From 77bad2c94604678570d65cdf5adf1c9b9a62ab0f Mon Sep 17 00:00:00 2001 From: Paul Stack Date: Tue, 10 Nov 2015 20:59:01 +0000 Subject: [PATCH 6/6] Changes to Aws Kinesis Firehouse Docs Changing the AWS Kinesis Firehose docs as per @jen20's feedback --- .../aws/r/kinesis_firehose_delivery_stream.html.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown index 1eec58951..61a6c62b5 100644 --- a/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/source/docs/providers/aws/r/kinesis_firehose_delivery_stream.html.markdown @@ -47,7 +47,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { } ``` -~> **NOTE:** Kinesis Firehose is currently only support in us-east-1, us-west-2 and eu-west-1. This implementation of Kinesis Firehose only supports the s3 destination type +~> **NOTE:** Kinesis Firehose is currently only supported in us-east-1, us-west-2 and eu-west-1. This implementation of Kinesis Firehose only supports the s3 destination type as Terraform doesn't support Redshift yet. ## Argument Reference