From fc983c5505c8f332f7f199afa6fb48222c88f18a Mon Sep 17 00:00:00 2001 From: stack72 Date: Mon, 9 Nov 2015 22:26:55 +0000 Subject: [PATCH] Initial Create, Read and Delete work for the S3 part of the Kinesis Firehose resource --- builtin/providers/aws/config.go | 5 + builtin/providers/aws/provider.go | 203 ++++++++--------- .../resource_aws_kinesis_firehose_stream.go | 210 ++++++++++++++++++ 3 files changed, 317 insertions(+), 101 deletions(-) create mode 100644 builtin/providers/aws/resource_aws_kinesis_firehose_stream.go diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index 3e835c106..bd7b08924 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -27,6 +27,7 @@ import ( "github.com/aws/aws-sdk-go/service/elasticache" elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/firehose" "github.com/aws/aws-sdk-go/service/glacier" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/kinesis" @@ -74,6 +75,7 @@ type AWSClient struct { rdsconn *rds.RDS iamconn *iam.IAM kinesisconn *kinesis.Kinesis + firehoseconn *firehose.Firehose elasticacheconn *elasticache.ElastiCache lambdaconn *lambda.Lambda opsworksconn *opsworks.OpsWorks @@ -168,6 +170,9 @@ func (c *Config) Client() (interface{}, error) { errs = append(errs, authErr) } + log.Println("[INFO] Initializing Kinesis Firehose Connection") + client.firehoseconn = autoscaling.New(sess) + log.Println("[INFO] Initializing AutoScaling connection") client.autoscalingconn = autoscaling.New(sess) diff --git a/builtin/providers/aws/provider.go b/builtin/providers/aws/provider.go index b5392429a..dfa1565fc 100644 --- a/builtin/providers/aws/provider.go +++ b/builtin/providers/aws/provider.go @@ -163,107 +163,108 @@ func Provider() terraform.ResourceProvider { }, ResourcesMap: map[string]*schema.Resource{ - "aws_ami": resourceAwsAmi(), - "aws_ami_copy": resourceAwsAmiCopy(), - "aws_ami_from_instance": resourceAwsAmiFromInstance(), - "aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(), - "aws_autoscaling_group": resourceAwsAutoscalingGroup(), - "aws_autoscaling_notification": resourceAwsAutoscalingNotification(), - "aws_autoscaling_policy": resourceAwsAutoscalingPolicy(), - "aws_cloudformation_stack": resourceAwsCloudFormationStack(), - "aws_cloudtrail": resourceAwsCloudTrail(), - "aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(), - "aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(), - "aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(), - "aws_codedeploy_app": resourceAwsCodeDeployApp(), - "aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(), - "aws_codecommit_repository": resourceAwsCodeCommitRepository(), - "aws_customer_gateway": resourceAwsCustomerGateway(), - "aws_db_instance": resourceAwsDbInstance(), - "aws_db_parameter_group": resourceAwsDbParameterGroup(), - "aws_db_security_group": resourceAwsDbSecurityGroup(), - "aws_db_subnet_group": resourceAwsDbSubnetGroup(), - "aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(), - "aws_dynamodb_table": resourceAwsDynamoDbTable(), - "aws_ebs_volume": resourceAwsEbsVolume(), - "aws_ecs_cluster": resourceAwsEcsCluster(), - "aws_ecs_service": resourceAwsEcsService(), - "aws_ecs_task_definition": resourceAwsEcsTaskDefinition(), - "aws_efs_file_system": resourceAwsEfsFileSystem(), - "aws_efs_mount_target": resourceAwsEfsMountTarget(), - "aws_eip": resourceAwsEip(), - "aws_elasticache_cluster": resourceAwsElasticacheCluster(), - "aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(), - "aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(), - "aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(), - "aws_elasticsearch_domain": resourceAwsElasticSearchDomain(), - "aws_elb": resourceAwsElb(), - "aws_flow_log": resourceAwsFlowLog(), - "aws_glacier_vault": resourceAwsGlacierVault(), - "aws_iam_access_key": resourceAwsIamAccessKey(), - "aws_iam_group_policy": resourceAwsIamGroupPolicy(), - "aws_iam_group": resourceAwsIamGroup(), - "aws_iam_group_membership": resourceAwsIamGroupMembership(), - "aws_iam_instance_profile": resourceAwsIamInstanceProfile(), - "aws_iam_policy": resourceAwsIamPolicy(), - "aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(), - "aws_iam_role_policy": resourceAwsIamRolePolicy(), - "aws_iam_role": resourceAwsIamRole(), - "aws_iam_saml_provider": resourceAwsIamSamlProvider(), - "aws_iam_server_certificate": resourceAwsIAMServerCertificate(), - "aws_iam_user_policy": resourceAwsIamUserPolicy(), - "aws_iam_user": resourceAwsIamUser(), - "aws_instance": resourceAwsInstance(), - "aws_internet_gateway": resourceAwsInternetGateway(), - "aws_key_pair": resourceAwsKeyPair(), - "aws_kinesis_stream": resourceAwsKinesisStream(), - "aws_lambda_function": resourceAwsLambdaFunction(), - "aws_launch_configuration": resourceAwsLaunchConfiguration(), - "aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(), - "aws_main_route_table_association": resourceAwsMainRouteTableAssociation(), - "aws_network_acl": resourceAwsNetworkAcl(), - "aws_network_interface": resourceAwsNetworkInterface(), - "aws_opsworks_stack": resourceAwsOpsworksStack(), - "aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(), - "aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(), - "aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(), - "aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(), - "aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(), - "aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(), - "aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(), - "aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(), - "aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(), - "aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(), - "aws_placement_group": resourceAwsPlacementGroup(), - "aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(), - "aws_rds_cluster": resourceAwsRDSCluster(), - "aws_rds_cluster_instance": resourceAwsRDSClusterInstance(), - "aws_route53_delegation_set": resourceAwsRoute53DelegationSet(), - "aws_route53_record": resourceAwsRoute53Record(), - "aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(), - "aws_route53_zone": resourceAwsRoute53Zone(), - "aws_route53_health_check": resourceAwsRoute53HealthCheck(), - "aws_route": resourceAwsRoute(), - "aws_route_table": resourceAwsRouteTable(), - "aws_route_table_association": resourceAwsRouteTableAssociation(), - "aws_s3_bucket": resourceAwsS3Bucket(), - "aws_s3_bucket_object": resourceAwsS3BucketObject(), - "aws_security_group": resourceAwsSecurityGroup(), - "aws_security_group_rule": resourceAwsSecurityGroupRule(), - "aws_spot_instance_request": resourceAwsSpotInstanceRequest(), - "aws_sqs_queue": resourceAwsSqsQueue(), - "aws_sns_topic": resourceAwsSnsTopic(), - "aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(), - "aws_subnet": resourceAwsSubnet(), - "aws_volume_attachment": resourceAwsVolumeAttachment(), - "aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(), - "aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(), - "aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(), - "aws_vpc": resourceAwsVpc(), - "aws_vpc_endpoint": resourceAwsVpcEndpoint(), - "aws_vpn_connection": resourceAwsVpnConnection(), - "aws_vpn_connection_route": resourceAwsVpnConnectionRoute(), - "aws_vpn_gateway": resourceAwsVpnGateway(), + "aws_ami": resourceAwsAmi(), + "aws_ami_copy": resourceAwsAmiCopy(), + "aws_ami_from_instance": resourceAwsAmiFromInstance(), + "aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(), + "aws_autoscaling_group": resourceAwsAutoscalingGroup(), + "aws_autoscaling_notification": resourceAwsAutoscalingNotification(), + "aws_autoscaling_policy": resourceAwsAutoscalingPolicy(), + "aws_cloudformation_stack": resourceAwsCloudFormationStack(), + "aws_cloudtrail": resourceAwsCloudTrail(), + "aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(), + "aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(), + "aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(), + "aws_codedeploy_app": resourceAwsCodeDeployApp(), + "aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(), + "aws_codecommit_repository": resourceAwsCodeCommitRepository(), + "aws_customer_gateway": resourceAwsCustomerGateway(), + "aws_db_instance": resourceAwsDbInstance(), + "aws_db_parameter_group": resourceAwsDbParameterGroup(), + "aws_db_security_group": resourceAwsDbSecurityGroup(), + "aws_db_subnet_group": resourceAwsDbSubnetGroup(), + "aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(), + "aws_dynamodb_table": resourceAwsDynamoDbTable(), + "aws_ebs_volume": resourceAwsEbsVolume(), + "aws_ecs_cluster": resourceAwsEcsCluster(), + "aws_ecs_service": resourceAwsEcsService(), + "aws_ecs_task_definition": resourceAwsEcsTaskDefinition(), + "aws_efs_file_system": resourceAwsEfsFileSystem(), + "aws_efs_mount_target": resourceAwsEfsMountTarget(), + "aws_eip": resourceAwsEip(), + "aws_elasticache_cluster": resourceAwsElasticacheCluster(), + "aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(), + "aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(), + "aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(), + "aws_elasticsearch_domain": resourceAwsElasticSearchDomain(), + "aws_elb": resourceAwsElb(), + "aws_flow_log": resourceAwsFlowLog(), + "aws_glacier_vault": resourceAwsGlacierVault(), + "aws_iam_access_key": resourceAwsIamAccessKey(), + "aws_iam_group_policy": resourceAwsIamGroupPolicy(), + "aws_iam_group": resourceAwsIamGroup(), + "aws_iam_group_membership": resourceAwsIamGroupMembership(), + "aws_iam_instance_profile": resourceAwsIamInstanceProfile(), + "aws_iam_policy": resourceAwsIamPolicy(), + "aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(), + "aws_iam_role_policy": resourceAwsIamRolePolicy(), + "aws_iam_role": resourceAwsIamRole(), + "aws_iam_saml_provider": resourceAwsIamSamlProvider(), + "aws_iam_server_certificate": resourceAwsIAMServerCertificate(), + "aws_iam_user_policy": resourceAwsIamUserPolicy(), + "aws_iam_user": resourceAwsIamUser(), + "aws_instance": resourceAwsInstance(), + "aws_internet_gateway": resourceAwsInternetGateway(), + "aws_key_pair": resourceAwsKeyPair(), + "aws_kinesis_firehose_delivery_stream": resourceAwsKinesisFirehoseDeliveryStream(), + "aws_kinesis_stream": resourceAwsKinesisStream(), + "aws_lambda_function": resourceAwsLambdaFunction(), + "aws_launch_configuration": resourceAwsLaunchConfiguration(), + "aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(), + "aws_main_route_table_association": resourceAwsMainRouteTableAssociation(), + "aws_network_acl": resourceAwsNetworkAcl(), + "aws_network_interface": resourceAwsNetworkInterface(), + "aws_opsworks_stack": resourceAwsOpsworksStack(), + "aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(), + "aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(), + "aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(), + "aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(), + "aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(), + "aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(), + "aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(), + "aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(), + "aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(), + "aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(), + "aws_placement_group": resourceAwsPlacementGroup(), + "aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(), + "aws_rds_cluster": resourceAwsRDSCluster(), + "aws_rds_cluster_instance": resourceAwsRDSClusterInstance(), + "aws_route53_delegation_set": resourceAwsRoute53DelegationSet(), + "aws_route53_record": resourceAwsRoute53Record(), + "aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(), + "aws_route53_zone": resourceAwsRoute53Zone(), + "aws_route53_health_check": resourceAwsRoute53HealthCheck(), + "aws_route": resourceAwsRoute(), + "aws_route_table": resourceAwsRouteTable(), + "aws_route_table_association": resourceAwsRouteTableAssociation(), + "aws_s3_bucket": resourceAwsS3Bucket(), + "aws_s3_bucket_object": resourceAwsS3BucketObject(), + "aws_security_group": resourceAwsSecurityGroup(), + "aws_security_group_rule": resourceAwsSecurityGroupRule(), + "aws_spot_instance_request": resourceAwsSpotInstanceRequest(), + "aws_sqs_queue": resourceAwsSqsQueue(), + "aws_sns_topic": resourceAwsSnsTopic(), + "aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(), + "aws_subnet": resourceAwsSubnet(), + "aws_volume_attachment": resourceAwsVolumeAttachment(), + "aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(), + "aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(), + "aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(), + "aws_vpc": resourceAwsVpc(), + "aws_vpc_endpoint": resourceAwsVpcEndpoint(), + "aws_vpn_connection": resourceAwsVpnConnection(), + "aws_vpn_connection_route": resourceAwsVpnConnectionRoute(), + "aws_vpn_gateway": resourceAwsVpnGateway(), }, ConfigureFunc: providerConfigure, diff --git a/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go b/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go new file mode 100644 index 000000000..320e078b2 --- /dev/null +++ b/builtin/providers/aws/resource_aws_kinesis_firehose_stream.go @@ -0,0 +1,210 @@ +package aws + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/firehose" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsKinesisFirehoseDeliveryStreamCreate, + Read: resourceAwsKinesisFirehoseDeliveryStreamRead, + Update: resourceAwsKinesisFirehoseDeliveryStreamUpdate, + Delete: resourceAwsKinesisFirehoseDeliveryStreamDelete, + + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "destination": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "role_arn": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + + "s3_bucket_arn": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + + "s3_prefix": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + + "s3_buffer_size": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 5, + }, + + "s3_buffer_interval": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 300, + }, + + "s3_data_compression": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "UNCOMPRESSED", + }, + + "arn": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + }, + } +} + +func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + + sn := d.Get("name").(string) + input := &firehose.CreateDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + } + + s3_config := &firehose.S3DestinationConfiguration{ + BucketARN: aws.String(d.Get("s3_bucket_arn").(string)), + RoleARN: aws.String(d.Get("role_arn").(string)), + BufferingHints: &firehose.BufferingHints{ + IntervalInSeconds: aws.Int64(int64(d.Get("s3_buffer_interval").(int))), + SizeInMBs: aws.Int64(int64(d.Get("s3_buffer_size").(int))), + }, + CompressionFormat: aws.String(d.Get("s3_data_compression").(string)), + } + if v, ok := d.GetOk("s3_prefix"); ok { + s3_config.Prefix = aws.String(v.(string)) + } + + input.S3DestinationConfiguration = s3_config + + _, err := conn.CreateDeliveryStream(input) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) + } + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{"CREATING"}, + Target: "ACTIVE", + Refresh: firehoseStreamStateRefreshFunc(conn, sn), + Timeout: 5 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + firehoseStream, err := stateConf.WaitForState() + if err != nil { + return fmt.Errorf( + "Error waiting for Kinesis Stream (%s) to become active: %s", + sn, err) + } + + s := firehoseStream.(*firehose.DeliveryStreamDescription) + d.SetId(*s.DeliveryStreamARN) + d.Set("arn", s.DeliveryStreamARN) + + return resourceAwsKinesisStreamUpdate(d, meta) +} + +func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta interface{}) error { + // conn := meta.(*AWSClient).firehoseconn + + return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) +} + +func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + sn := d.Get("name").(string) + describeOpts := &firehose.DeliveryStreamDescription{ + DeliveryStreamName: aws.String(sn), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "ResourceNotFoundException" { + d.SetId("") + return nil + } + return fmt.Errorf("[WARN] Error reading Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) + } + return err + } + + s := resp.DeliveryStreamDescription + d.Set("arn", *s.DeliveryStreamARN) + + return nil +} + +func resourceAwsKinesisFirehoseDeliveryStreamDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).firehoseconn + + sn := d.Get("name").(string) + _, err := conn.DeleteDeliveryStream(&firehose.DeleteDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + }) + + if err != nil { + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{"DELETING"}, + Target: "DESTROYED", + Refresh: firehoseStreamStateRefreshFunc(conn, sn), + Timeout: 5 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + _, err = stateConf.WaitForState() + if err != nil { + return fmt.Errorf( + "Error waiting for Delivery Stream (%s) to be destroyed: %s", + sn, err) + } + + d.SetId("") + return nil +} + +func firehoseStreamStateRefreshFunc(conn *firehose.Firehose, sn string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + describeOpts := &firehose.DescribeDeliveryStreamInput{ + DeliveryStreamName: aws.String(sn), + } + resp, err := conn.DescribeDeliveryStream(describeOpts) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == "ResourceNotFoundException" { + return 42, "DESTROYED", nil + } + return nil, awsErr.Code(), err + } + return nil, "failed", err + } + + return resp.DeliveryStreamDescription, *resp.DeliveryStreamDescription.DeliveryStreamStatus, nil + } +}