provider/aws: Update `aws_emr_cluster` to support Security Configuration (#14133)

* added emr security configurations

* gofmt after rebase

* provider/aws: Update EMR Cluster to support Security Configuration

* update test to create key

* update docs
This commit is contained in:
Clint 2017-05-01 17:02:59 -05:00 committed by GitHub
parent 08e778c883
commit a8e4a9bf07
3 changed files with 377 additions and 0 deletions

View File

@ -157,6 +157,11 @@ func resourceAwsEMRCluster() *schema.Resource {
ForceNew: true, ForceNew: true,
Required: true, Required: true,
}, },
"security_configuration": {
Type: schema.TypeString,
ForceNew: true,
Optional: true,
},
"autoscaling_role": &schema.Schema{ "autoscaling_role": &schema.Schema{
Type: schema.TypeString, Type: schema.TypeString,
ForceNew: true, ForceNew: true,
@ -268,6 +273,10 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error
params.AutoScalingRole = aws.String(v.(string)) params.AutoScalingRole = aws.String(v.(string))
} }
if v, ok := d.GetOk("security_configuration"); ok {
params.SecurityConfiguration = aws.String(v.(string))
}
if instanceProfile != "" { if instanceProfile != "" {
params.JobFlowRole = aws.String(instanceProfile) params.JobFlowRole = aws.String(instanceProfile)
} }
@ -361,6 +370,7 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error {
d.Set("name", cluster.Name) d.Set("name", cluster.Name)
d.Set("service_role", cluster.ServiceRole) d.Set("service_role", cluster.ServiceRole)
d.Set("security_configuration", cluster.SecurityConfiguration)
d.Set("autoscaling_role", cluster.AutoScalingRole) d.Set("autoscaling_role", cluster.AutoScalingRole)
d.Set("release_label", cluster.ReleaseLabel) d.Set("release_label", cluster.ReleaseLabel)
d.Set("log_uri", cluster.LogUri) d.Set("log_uri", cluster.LogUri)

View File

@ -30,6 +30,22 @@ func TestAccAWSEMRCluster_basic(t *testing.T) {
}) })
} }
func TestAccAWSEMRCluster_security_config(t *testing.T) {
var cluster emr.Cluster
r := acctest.RandInt()
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSEmrDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSEmrClusterConfig_SecurityConfiguration(r),
Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
},
},
})
}
func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) { func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) {
var cluster emr.Cluster var cluster emr.Cluster
rName := acctest.RandomWithPrefix("tf-emr-bootstrap") rName := acctest.RandomWithPrefix("tf-emr-bootstrap")
@ -881,6 +897,356 @@ resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" {
`, r, r, r, r, r, r, r, r, r, r) `, r, r, r, r, r, r, r, r, r, r)
} }
func testAccAWSEmrClusterConfig_SecurityConfiguration(r int) string {
return fmt.Sprintf(`
provider "aws" {
region = "us-west-2"
}
resource "aws_emr_cluster" "tf-test-cluster" {
name = "emr-test-%d"
release_label = "emr-5.5.0"
applications = ["Spark"]
ec2_attributes {
subnet_id = "${aws_subnet.main.id}"
emr_managed_master_security_group = "${aws_security_group.allow_all.id}"
emr_managed_slave_security_group = "${aws_security_group.allow_all.id}"
instance_profile = "${aws_iam_instance_profile.emr_profile.arn}"
}
master_instance_type = "m3.xlarge"
core_instance_type = "m3.xlarge"
core_instance_count = 1
security_configuration = "${aws_emr_security_configuration.foo.name}"
tags {
role = "rolename"
dns_zone = "env_zone"
env = "env"
name = "name-env"
}
keep_job_flow_alive_when_no_steps = true
termination_protection = false
bootstrap_action {
path = "s3://elasticmapreduce/bootstrap-actions/run-if"
name = "runif"
args = ["instance.isMaster=true", "echo running on master node"]
}
configurations = "test-fixtures/emr_configurations.json"
depends_on = ["aws_main_route_table_association.a"]
service_role = "${aws_iam_role.iam_emr_default_role.arn}"
autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}"
}
resource "aws_security_group" "allow_all" {
name = "allow_all_%d"
description = "Allow all inbound traffic"
vpc_id = "${aws_vpc.main.id}"
ingress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
depends_on = ["aws_subnet.main"]
lifecycle {
ignore_changes = ["ingress", "egress"]
}
tags {
name = "emr_test"
}
}
resource "aws_vpc" "main" {
cidr_block = "168.31.0.0/16"
enable_dns_hostnames = true
tags {
name = "emr_test_%d"
}
}
resource "aws_subnet" "main" {
vpc_id = "${aws_vpc.main.id}"
cidr_block = "168.31.0.0/20"
tags {
name = "emr_test_%d"
}
}
resource "aws_internet_gateway" "gw" {
vpc_id = "${aws_vpc.main.id}"
}
resource "aws_route_table" "r" {
vpc_id = "${aws_vpc.main.id}"
route {
cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.gw.id}"
}
}
resource "aws_main_route_table_association" "a" {
vpc_id = "${aws_vpc.main.id}"
route_table_id = "${aws_route_table.r.id}"
}
###
# IAM things
###
# IAM role for EMR Service
resource "aws_iam_role" "iam_emr_default_role" {
name = "iam_emr_default_role_%d"
assume_role_policy = <<EOT
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOT
}
resource "aws_iam_role_policy_attachment" "service-attach" {
role = "${aws_iam_role.iam_emr_default_role.id}"
policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}"
}
resource "aws_iam_policy" "iam_emr_default_policy" {
name = "iam_emr_default_policy_%d"
policy = <<EOT
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"ec2:DescribeVolumes",
"ec2:DetachVolume",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
EOT
}
# IAM Role for EC2 Instance Profile
resource "aws_iam_role" "iam_emr_profile_role" {
name = "iam_emr_profile_role_%d"
assume_role_policy = <<EOT
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOT
}
resource "aws_iam_instance_profile" "emr_profile" {
name = "emr_profile_%d"
roles = ["${aws_iam_role.iam_emr_profile_role.name}"]
}
resource "aws_iam_role_policy_attachment" "profile-attach" {
role = "${aws_iam_role.iam_emr_profile_role.id}"
policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}"
}
resource "aws_iam_policy" "iam_emr_profile_policy" {
name = "iam_emr_profile_policy_%d"
policy = <<EOT
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
EOT
}
# IAM Role for autoscaling
resource "aws_iam_role" "emr-autoscaling-role" {
name = "EMR_AutoScaling_DefaultRole_%d"
assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}"
}
data "aws_iam_policy_document" "emr-autoscaling-role-policy" {
statement {
effect = "Allow"
actions = ["sts:AssumeRole"]
principals = {
type = "Service"
identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"]
}
}
}
resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" {
role = "${aws_iam_role.emr-autoscaling-role.name}"
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole"
}
resource "aws_emr_security_configuration" "foo" {
configuration = <<EOF
{
"EncryptionConfiguration": {
"AtRestEncryptionConfiguration": {
"S3EncryptionConfiguration": {
"EncryptionMode": "SSE-S3"
},
"LocalDiskEncryptionConfiguration": {
"EncryptionKeyProviderType": "AwsKms",
"AwsKmsKey": "${aws_kms_key.foo.arn}"
}
},
"EnableInTransitEncryption": false,
"EnableAtRestEncryption": true
}
}
EOF
}
resource "aws_kms_key" "foo" {
description = "Terraform acc test %d"
deletion_window_in_days = 7
policy = <<POLICY
{
"Version": "2012-10-17",
"Id": "kms-tf-1",
"Statement": [
{
"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": "kms:*",
"Resource": "*"
}
]
}
POLICY
}
`, r, r, r, r, r, r, r, r, r, r, r)
}
func testAccAWSEmrClusterConfigTerminationPolicyUpdated(r int) string { func testAccAWSEmrClusterConfigTerminationPolicyUpdated(r int) string {
return fmt.Sprintf(` return fmt.Sprintf(`
provider "aws" { provider "aws" {

View File

@ -67,6 +67,7 @@ The following arguments are supported:
* `release_label` - (Required) The release label for the Amazon EMR release * `release_label` - (Required) The release label for the Amazon EMR release
* `master_instance_type` - (Required) The EC2 instance type of the master node * `master_instance_type` - (Required) The EC2 instance type of the master node
* `service_role` - (Required) IAM role that will be assumed by the Amazon EMR service to access AWS resources * `service_role` - (Required) IAM role that will be assumed by the Amazon EMR service to access AWS resources
* `security_configuration` - (Optional) The security configuration name to attach to the EMR cluster. Only valid for EMR clusters with `release_label` 4.8.0 or greater
* `core_instance_type` - (Optional) The EC2 instance type of the slave nodes * `core_instance_type` - (Optional) The EC2 instance type of the slave nodes
* `core_instance_count` - (Optional) Number of Amazon EC2 instances used to execute the job flow. EMR will use one node as the cluster's master node and use the remainder of the nodes (`core_instance_count`-1) as core nodes. Default `1` * `core_instance_count` - (Optional) Number of Amazon EC2 instances used to execute the job flow. EMR will use one node as the cluster's master node and use the remainder of the nodes (`core_instance_count`-1) as core nodes. Default `1`
* `log_uri` - (Optional) S3 bucket to write the log files of the job flow. If a value * `log_uri` - (Optional) S3 bucket to write the log files of the job flow. If a value