Initial Create, Read and Delete work for the S3 part of the Kinesis Firehose resource

This commit is contained in:
stack72 2015-11-09 22:26:55 +00:00
parent 6906dc1b2b
commit fc983c5505
3 changed files with 317 additions and 101 deletions

View File

@ -27,6 +27,7 @@ import (
"github.com/aws/aws-sdk-go/service/elasticache"
elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice"
"github.com/aws/aws-sdk-go/service/elb"
"github.com/aws/aws-sdk-go/service/firehose"
"github.com/aws/aws-sdk-go/service/glacier"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/kinesis"
@ -74,6 +75,7 @@ type AWSClient struct {
rdsconn *rds.RDS
iamconn *iam.IAM
kinesisconn *kinesis.Kinesis
firehoseconn *firehose.Firehose
elasticacheconn *elasticache.ElastiCache
lambdaconn *lambda.Lambda
opsworksconn *opsworks.OpsWorks
@ -168,6 +170,9 @@ func (c *Config) Client() (interface{}, error) {
errs = append(errs, authErr)
}
log.Println("[INFO] Initializing Kinesis Firehose Connection")
client.firehoseconn = autoscaling.New(sess)
log.Println("[INFO] Initializing AutoScaling connection")
client.autoscalingconn = autoscaling.New(sess)

View File

@ -163,107 +163,108 @@ func Provider() terraform.ResourceProvider {
},
ResourcesMap: map[string]*schema.Resource{
"aws_ami": resourceAwsAmi(),
"aws_ami_copy": resourceAwsAmiCopy(),
"aws_ami_from_instance": resourceAwsAmiFromInstance(),
"aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(),
"aws_autoscaling_group": resourceAwsAutoscalingGroup(),
"aws_autoscaling_notification": resourceAwsAutoscalingNotification(),
"aws_autoscaling_policy": resourceAwsAutoscalingPolicy(),
"aws_cloudformation_stack": resourceAwsCloudFormationStack(),
"aws_cloudtrail": resourceAwsCloudTrail(),
"aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(),
"aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(),
"aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(),
"aws_codedeploy_app": resourceAwsCodeDeployApp(),
"aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(),
"aws_codecommit_repository": resourceAwsCodeCommitRepository(),
"aws_customer_gateway": resourceAwsCustomerGateway(),
"aws_db_instance": resourceAwsDbInstance(),
"aws_db_parameter_group": resourceAwsDbParameterGroup(),
"aws_db_security_group": resourceAwsDbSecurityGroup(),
"aws_db_subnet_group": resourceAwsDbSubnetGroup(),
"aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(),
"aws_dynamodb_table": resourceAwsDynamoDbTable(),
"aws_ebs_volume": resourceAwsEbsVolume(),
"aws_ecs_cluster": resourceAwsEcsCluster(),
"aws_ecs_service": resourceAwsEcsService(),
"aws_ecs_task_definition": resourceAwsEcsTaskDefinition(),
"aws_efs_file_system": resourceAwsEfsFileSystem(),
"aws_efs_mount_target": resourceAwsEfsMountTarget(),
"aws_eip": resourceAwsEip(),
"aws_elasticache_cluster": resourceAwsElasticacheCluster(),
"aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(),
"aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(),
"aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(),
"aws_elasticsearch_domain": resourceAwsElasticSearchDomain(),
"aws_elb": resourceAwsElb(),
"aws_flow_log": resourceAwsFlowLog(),
"aws_glacier_vault": resourceAwsGlacierVault(),
"aws_iam_access_key": resourceAwsIamAccessKey(),
"aws_iam_group_policy": resourceAwsIamGroupPolicy(),
"aws_iam_group": resourceAwsIamGroup(),
"aws_iam_group_membership": resourceAwsIamGroupMembership(),
"aws_iam_instance_profile": resourceAwsIamInstanceProfile(),
"aws_iam_policy": resourceAwsIamPolicy(),
"aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(),
"aws_iam_role_policy": resourceAwsIamRolePolicy(),
"aws_iam_role": resourceAwsIamRole(),
"aws_iam_saml_provider": resourceAwsIamSamlProvider(),
"aws_iam_server_certificate": resourceAwsIAMServerCertificate(),
"aws_iam_user_policy": resourceAwsIamUserPolicy(),
"aws_iam_user": resourceAwsIamUser(),
"aws_instance": resourceAwsInstance(),
"aws_internet_gateway": resourceAwsInternetGateway(),
"aws_key_pair": resourceAwsKeyPair(),
"aws_kinesis_stream": resourceAwsKinesisStream(),
"aws_lambda_function": resourceAwsLambdaFunction(),
"aws_launch_configuration": resourceAwsLaunchConfiguration(),
"aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(),
"aws_main_route_table_association": resourceAwsMainRouteTableAssociation(),
"aws_network_acl": resourceAwsNetworkAcl(),
"aws_network_interface": resourceAwsNetworkInterface(),
"aws_opsworks_stack": resourceAwsOpsworksStack(),
"aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(),
"aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(),
"aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(),
"aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(),
"aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(),
"aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(),
"aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(),
"aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(),
"aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(),
"aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(),
"aws_placement_group": resourceAwsPlacementGroup(),
"aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(),
"aws_rds_cluster": resourceAwsRDSCluster(),
"aws_rds_cluster_instance": resourceAwsRDSClusterInstance(),
"aws_route53_delegation_set": resourceAwsRoute53DelegationSet(),
"aws_route53_record": resourceAwsRoute53Record(),
"aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(),
"aws_route53_zone": resourceAwsRoute53Zone(),
"aws_route53_health_check": resourceAwsRoute53HealthCheck(),
"aws_route": resourceAwsRoute(),
"aws_route_table": resourceAwsRouteTable(),
"aws_route_table_association": resourceAwsRouteTableAssociation(),
"aws_s3_bucket": resourceAwsS3Bucket(),
"aws_s3_bucket_object": resourceAwsS3BucketObject(),
"aws_security_group": resourceAwsSecurityGroup(),
"aws_security_group_rule": resourceAwsSecurityGroupRule(),
"aws_spot_instance_request": resourceAwsSpotInstanceRequest(),
"aws_sqs_queue": resourceAwsSqsQueue(),
"aws_sns_topic": resourceAwsSnsTopic(),
"aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(),
"aws_subnet": resourceAwsSubnet(),
"aws_volume_attachment": resourceAwsVolumeAttachment(),
"aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(),
"aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(),
"aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(),
"aws_vpc": resourceAwsVpc(),
"aws_vpc_endpoint": resourceAwsVpcEndpoint(),
"aws_vpn_connection": resourceAwsVpnConnection(),
"aws_vpn_connection_route": resourceAwsVpnConnectionRoute(),
"aws_vpn_gateway": resourceAwsVpnGateway(),
"aws_ami": resourceAwsAmi(),
"aws_ami_copy": resourceAwsAmiCopy(),
"aws_ami_from_instance": resourceAwsAmiFromInstance(),
"aws_app_cookie_stickiness_policy": resourceAwsAppCookieStickinessPolicy(),
"aws_autoscaling_group": resourceAwsAutoscalingGroup(),
"aws_autoscaling_notification": resourceAwsAutoscalingNotification(),
"aws_autoscaling_policy": resourceAwsAutoscalingPolicy(),
"aws_cloudformation_stack": resourceAwsCloudFormationStack(),
"aws_cloudtrail": resourceAwsCloudTrail(),
"aws_cloudwatch_log_group": resourceAwsCloudWatchLogGroup(),
"aws_autoscaling_lifecycle_hook": resourceAwsAutoscalingLifecycleHook(),
"aws_cloudwatch_metric_alarm": resourceAwsCloudWatchMetricAlarm(),
"aws_codedeploy_app": resourceAwsCodeDeployApp(),
"aws_codedeploy_deployment_group": resourceAwsCodeDeployDeploymentGroup(),
"aws_codecommit_repository": resourceAwsCodeCommitRepository(),
"aws_customer_gateway": resourceAwsCustomerGateway(),
"aws_db_instance": resourceAwsDbInstance(),
"aws_db_parameter_group": resourceAwsDbParameterGroup(),
"aws_db_security_group": resourceAwsDbSecurityGroup(),
"aws_db_subnet_group": resourceAwsDbSubnetGroup(),
"aws_directory_service_directory": resourceAwsDirectoryServiceDirectory(),
"aws_dynamodb_table": resourceAwsDynamoDbTable(),
"aws_ebs_volume": resourceAwsEbsVolume(),
"aws_ecs_cluster": resourceAwsEcsCluster(),
"aws_ecs_service": resourceAwsEcsService(),
"aws_ecs_task_definition": resourceAwsEcsTaskDefinition(),
"aws_efs_file_system": resourceAwsEfsFileSystem(),
"aws_efs_mount_target": resourceAwsEfsMountTarget(),
"aws_eip": resourceAwsEip(),
"aws_elasticache_cluster": resourceAwsElasticacheCluster(),
"aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(),
"aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(),
"aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(),
"aws_elasticsearch_domain": resourceAwsElasticSearchDomain(),
"aws_elb": resourceAwsElb(),
"aws_flow_log": resourceAwsFlowLog(),
"aws_glacier_vault": resourceAwsGlacierVault(),
"aws_iam_access_key": resourceAwsIamAccessKey(),
"aws_iam_group_policy": resourceAwsIamGroupPolicy(),
"aws_iam_group": resourceAwsIamGroup(),
"aws_iam_group_membership": resourceAwsIamGroupMembership(),
"aws_iam_instance_profile": resourceAwsIamInstanceProfile(),
"aws_iam_policy": resourceAwsIamPolicy(),
"aws_iam_policy_attachment": resourceAwsIamPolicyAttachment(),
"aws_iam_role_policy": resourceAwsIamRolePolicy(),
"aws_iam_role": resourceAwsIamRole(),
"aws_iam_saml_provider": resourceAwsIamSamlProvider(),
"aws_iam_server_certificate": resourceAwsIAMServerCertificate(),
"aws_iam_user_policy": resourceAwsIamUserPolicy(),
"aws_iam_user": resourceAwsIamUser(),
"aws_instance": resourceAwsInstance(),
"aws_internet_gateway": resourceAwsInternetGateway(),
"aws_key_pair": resourceAwsKeyPair(),
"aws_kinesis_firehose_delivery_stream": resourceAwsKinesisFirehoseDeliveryStream(),
"aws_kinesis_stream": resourceAwsKinesisStream(),
"aws_lambda_function": resourceAwsLambdaFunction(),
"aws_launch_configuration": resourceAwsLaunchConfiguration(),
"aws_lb_cookie_stickiness_policy": resourceAwsLBCookieStickinessPolicy(),
"aws_main_route_table_association": resourceAwsMainRouteTableAssociation(),
"aws_network_acl": resourceAwsNetworkAcl(),
"aws_network_interface": resourceAwsNetworkInterface(),
"aws_opsworks_stack": resourceAwsOpsworksStack(),
"aws_opsworks_java_app_layer": resourceAwsOpsworksJavaAppLayer(),
"aws_opsworks_haproxy_layer": resourceAwsOpsworksHaproxyLayer(),
"aws_opsworks_static_web_layer": resourceAwsOpsworksStaticWebLayer(),
"aws_opsworks_php_app_layer": resourceAwsOpsworksPhpAppLayer(),
"aws_opsworks_rails_app_layer": resourceAwsOpsworksRailsAppLayer(),
"aws_opsworks_nodejs_app_layer": resourceAwsOpsworksNodejsAppLayer(),
"aws_opsworks_memcached_layer": resourceAwsOpsworksMemcachedLayer(),
"aws_opsworks_mysql_layer": resourceAwsOpsworksMysqlLayer(),
"aws_opsworks_ganglia_layer": resourceAwsOpsworksGangliaLayer(),
"aws_opsworks_custom_layer": resourceAwsOpsworksCustomLayer(),
"aws_placement_group": resourceAwsPlacementGroup(),
"aws_proxy_protocol_policy": resourceAwsProxyProtocolPolicy(),
"aws_rds_cluster": resourceAwsRDSCluster(),
"aws_rds_cluster_instance": resourceAwsRDSClusterInstance(),
"aws_route53_delegation_set": resourceAwsRoute53DelegationSet(),
"aws_route53_record": resourceAwsRoute53Record(),
"aws_route53_zone_association": resourceAwsRoute53ZoneAssociation(),
"aws_route53_zone": resourceAwsRoute53Zone(),
"aws_route53_health_check": resourceAwsRoute53HealthCheck(),
"aws_route": resourceAwsRoute(),
"aws_route_table": resourceAwsRouteTable(),
"aws_route_table_association": resourceAwsRouteTableAssociation(),
"aws_s3_bucket": resourceAwsS3Bucket(),
"aws_s3_bucket_object": resourceAwsS3BucketObject(),
"aws_security_group": resourceAwsSecurityGroup(),
"aws_security_group_rule": resourceAwsSecurityGroupRule(),
"aws_spot_instance_request": resourceAwsSpotInstanceRequest(),
"aws_sqs_queue": resourceAwsSqsQueue(),
"aws_sns_topic": resourceAwsSnsTopic(),
"aws_sns_topic_subscription": resourceAwsSnsTopicSubscription(),
"aws_subnet": resourceAwsSubnet(),
"aws_volume_attachment": resourceAwsVolumeAttachment(),
"aws_vpc_dhcp_options_association": resourceAwsVpcDhcpOptionsAssociation(),
"aws_vpc_dhcp_options": resourceAwsVpcDhcpOptions(),
"aws_vpc_peering_connection": resourceAwsVpcPeeringConnection(),
"aws_vpc": resourceAwsVpc(),
"aws_vpc_endpoint": resourceAwsVpcEndpoint(),
"aws_vpn_connection": resourceAwsVpnConnection(),
"aws_vpn_connection_route": resourceAwsVpnConnectionRoute(),
"aws_vpn_gateway": resourceAwsVpnGateway(),
},
ConfigureFunc: providerConfigure,

View File

@ -0,0 +1,210 @@
package aws
import (
"fmt"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/firehose"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/helper/schema"
)
func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource {
return &schema.Resource{
Create: resourceAwsKinesisFirehoseDeliveryStreamCreate,
Read: resourceAwsKinesisFirehoseDeliveryStreamRead,
Update: resourceAwsKinesisFirehoseDeliveryStreamUpdate,
Delete: resourceAwsKinesisFirehoseDeliveryStreamDelete,
Schema: map[string]*schema.Schema{
"name": &schema.Schema{
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"destination": &schema.Schema{
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"role_arn": &schema.Schema{
Type: schema.TypeString,
Required: true,
},
"s3_bucket_arn": &schema.Schema{
Type: schema.TypeString,
Required: true,
},
"s3_prefix": &schema.Schema{
Type: schema.TypeString,
Optional: true,
},
"s3_buffer_size": &schema.Schema{
Type: schema.TypeInt,
Optional: true,
Default: 5,
},
"s3_buffer_interval": &schema.Schema{
Type: schema.TypeInt,
Optional: true,
Default: 300,
},
"s3_data_compression": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Default: "UNCOMPRESSED",
},
"arn": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Computed: true,
},
},
}
}
func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).firehoseconn
sn := d.Get("name").(string)
input := &firehose.CreateDeliveryStreamInput{
DeliveryStreamName: aws.String(sn),
}
s3_config := &firehose.S3DestinationConfiguration{
BucketARN: aws.String(d.Get("s3_bucket_arn").(string)),
RoleARN: aws.String(d.Get("role_arn").(string)),
BufferingHints: &firehose.BufferingHints{
IntervalInSeconds: aws.Int64(int64(d.Get("s3_buffer_interval").(int))),
SizeInMBs: aws.Int64(int64(d.Get("s3_buffer_size").(int))),
},
CompressionFormat: aws.String(d.Get("s3_data_compression").(string)),
}
if v, ok := d.GetOk("s3_prefix"); ok {
s3_config.Prefix = aws.String(v.(string))
}
input.S3DestinationConfiguration = s3_config
_, err := conn.CreateDeliveryStream(input)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code())
}
return err
}
stateConf := &resource.StateChangeConf{
Pending: []string{"CREATING"},
Target: "ACTIVE",
Refresh: firehoseStreamStateRefreshFunc(conn, sn),
Timeout: 5 * time.Minute,
Delay: 10 * time.Second,
MinTimeout: 3 * time.Second,
}
firehoseStream, err := stateConf.WaitForState()
if err != nil {
return fmt.Errorf(
"Error waiting for Kinesis Stream (%s) to become active: %s",
sn, err)
}
s := firehoseStream.(*firehose.DeliveryStreamDescription)
d.SetId(*s.DeliveryStreamARN)
d.Set("arn", s.DeliveryStreamARN)
return resourceAwsKinesisStreamUpdate(d, meta)
}
func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta interface{}) error {
// conn := meta.(*AWSClient).firehoseconn
return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta)
}
func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).firehoseconn
sn := d.Get("name").(string)
describeOpts := &firehose.DeliveryStreamDescription{
DeliveryStreamName: aws.String(sn),
}
resp, err := conn.DescribeDeliveryStream(describeOpts)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ResourceNotFoundException" {
d.SetId("")
return nil
}
return fmt.Errorf("[WARN] Error reading Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code())
}
return err
}
s := resp.DeliveryStreamDescription
d.Set("arn", *s.DeliveryStreamARN)
return nil
}
func resourceAwsKinesisFirehoseDeliveryStreamDelete(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).firehoseconn
sn := d.Get("name").(string)
_, err := conn.DeleteDeliveryStream(&firehose.DeleteDeliveryStreamInput{
DeliveryStreamName: aws.String(sn),
})
if err != nil {
return err
}
stateConf := &resource.StateChangeConf{
Pending: []string{"DELETING"},
Target: "DESTROYED",
Refresh: firehoseStreamStateRefreshFunc(conn, sn),
Timeout: 5 * time.Minute,
Delay: 10 * time.Second,
MinTimeout: 3 * time.Second,
}
_, err = stateConf.WaitForState()
if err != nil {
return fmt.Errorf(
"Error waiting for Delivery Stream (%s) to be destroyed: %s",
sn, err)
}
d.SetId("")
return nil
}
func firehoseStreamStateRefreshFunc(conn *firehose.Firehose, sn string) resource.StateRefreshFunc {
return func() (interface{}, string, error) {
describeOpts := &firehose.DescribeDeliveryStreamInput{
DeliveryStreamName: aws.String(sn),
}
resp, err := conn.DescribeDeliveryStream(describeOpts)
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == "ResourceNotFoundException" {
return 42, "DESTROYED", nil
}
return nil, awsErr.Code(), err
}
return nil, "failed", err
}
return resp.DeliveryStreamDescription, *resp.DeliveryStreamDescription.DeliveryStreamStatus, nil
}
}