Add support for 'prefix_list_ids' to AWS VPC security group rules

Prefix list IDs are used when allowing egress to an AWS VPC Endpoint.

See http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-endpoints.html#vpc-endpoints-routing

builtin/providers/aws/resource_aws_security_group.go

@@ -153,6 +153,12 @@ func resourceAwsSecurityGroup() *schema.Resource {
 							Elem:     &schema.Schema{Type: schema.TypeString},
 						},
 
+						"prefix_list_ids": &schema.Schema{
+							Type:     schema.TypeList,
+							Optional: true,
+							Elem:     &schema.Schema{Type: schema.TypeString},
+						},
+
 						"security_groups": &schema.Schema{
 							Type:     schema.TypeSet,
 							Optional: true,
@@ -397,6 +403,18 @@ func resourceAwsSecurityGroupRuleHash(v interface{}) int {
 			buf.WriteString(fmt.Sprintf("%s-", v))
 		}
 	}
+	if v, ok := m["prefix_list_ids"]; ok {
+		vs := v.([]interface{})
+		s := make([]string, len(vs))
+		for i, raw := range vs {
+			s[i] = raw.(string)
+		}
+		sort.Strings(s)
+
+		for _, v := range s {
+			buf.WriteString(fmt.Sprintf("%s-", v))
+		}
+	}
 	if v, ok := m["security_groups"]; ok {
 		vs := v.(*schema.Set).List()
 		s := make([]string, len(vs))
@@ -449,6 +467,20 @@ func resourceAwsSecurityGroupIPPermGather(groupId string, permissions []*ec2.IpP
 			m["cidr_blocks"] = list
 		}
 
+		if len(perm.PrefixListIds) > 0 {
+			raw, ok := m["prefix_list_ids"]
+			if !ok {
+				raw = make([]string, 0, len(perm.PrefixListIds))
+			}
+			list := raw.([]string)
+
+			for _, pl := range perm.PrefixListIds {
+				list = append(list, *pl.PrefixListId)
+			}
+
+			m["prefix_list_ids"] = list
+		}
+
 		groups := flattenSecurityGroups(perm.UserIdGroupPairs, ownerId)
 		for i, g := range groups {
 			if *g.GroupId == groupId {
@@ -658,9 +690,10 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 			// local rule we're examining
 			rHash := idHash(rType, r["protocol"].(string), r["to_port"].(int64), r["from_port"].(int64), remoteSelfVal)
 			if rHash == localHash {
-				var numExpectedCidrs, numExpectedSGs, numRemoteCidrs, numRemoteSGs int
+				var numExpectedCidrs, numExpectedPrefixLists, numExpectedSGs, numRemoteCidrs, numRemotePrefixLists, numRemoteSGs int
 				var matchingCidrs []string
 				var matchingSGs []string
+				var matchingPrefixLists []string
 
 				// grab the local/remote cidr and sg groups, capturing the expected and
 				// actual counts
@@ -668,6 +701,10 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 				if ok {
 					numExpectedCidrs = len(l["cidr_blocks"].([]interface{}))
 				}
+				lpRaw, ok := l["prefix_list_ids"]
+				if ok {
+					numExpectedPrefixLists = len(l["prefix_list_ids"].([]interface{}))
+				}
 				lsRaw, ok := l["security_groups"]
 				if ok {
 					numExpectedSGs = len(l["security_groups"].(*schema.Set).List())
@@ -677,6 +714,10 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 				if ok {
 					numRemoteCidrs = len(r["cidr_blocks"].([]string))
 				}
+				rpRaw, ok := r["prefix_list_ids"]
+				if ok {
+					numRemotePrefixLists = len(r["prefix_list_ids"].([]string))
+				}
 
 				rsRaw, ok := r["security_groups"]
 				if ok {
@@ -688,6 +729,10 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 					log.Printf("[DEBUG] Local rule has more CIDR blocks, continuing (%d/%d)", numExpectedCidrs, numRemoteCidrs)
 					continue
 				}
+				if numExpectedPrefixLists > numRemotePrefixLists {
+					log.Printf("[DEBUG] Local rule has more prefix lists, continuing (%d/%d)", numExpectedPrefixLists, numRemotePrefixLists)
+					continue
+				}
 				if numExpectedSGs > numRemoteSGs {
 					log.Printf("[DEBUG] Local rule has more Security Groups, continuing (%d/%d)", numExpectedSGs, numRemoteSGs)
 					continue
@@ -721,6 +766,34 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 					}
 				}
 
+				// match prefix lists by converting both to sets, and using Set methods
+				var localPrefixLists []interface{}
+				if lpRaw != nil {
+					localPrefixLists = lpRaw.([]interface{})
+				}
+				localPrefixListsSet := schema.NewSet(schema.HashString, localPrefixLists)
+
+				// remote prefix lists are presented as a slice of strings, so we need to
+				// reformat them into a slice of interfaces to be used in creating the
+				// remote prefix list set
+				var remotePrefixLists []string
+				if rpRaw != nil {
+					remotePrefixLists = rpRaw.([]string)
+				}
+				// convert remote prefix lists to a set, for easy comparison
+				list = nil
+				for _, s := range remotePrefixLists {
+					list = append(list, s)
+				}
+				remotePrefixListsSet := schema.NewSet(schema.HashString, list)
+
+				// Build up a list of local prefix lists that are found in the remote set
+				for _, s := range localPrefixListsSet.List() {
+					if remotePrefixListsSet.Contains(s) {
+						matchingPrefixLists = append(matchingPrefixLists, s.(string))
+					}
+				}
+
 				// match SGs. Both local and remote are already sets
 				var localSGSet *schema.Set
 				if lsRaw == nil {
@@ -748,6 +821,7 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 				// match, and then remove those elements from the remote rule, so that
 				// this remote rule can still be considered by other local rules
 				if numExpectedCidrs == len(matchingCidrs) {
+					if numExpectedPrefixLists == len(matchingPrefixLists) {
 						if numExpectedSGs == len(matchingSGs) {
 							// confirm that self references match
 							var lSelf bool
@@ -774,6 +848,20 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 									delete(r, "cidr_blocks")
 								}
 
+								// pop local prefix lists from remote
+								diffPrefixLists := remotePrefixListsSet.Difference(localPrefixListsSet)
+								var newPrefixLists []string
+								for _, pRaw := range diffPrefixLists.List() {
+									newPrefixLists = append(newPrefixLists, pRaw.(string))
+								}
+
+								// reassigning
+								if len(newPrefixLists) > 0 {
+									r["prefix_list_ids"] = newPrefixLists
+								} else {
+									delete(r, "prefix_list_ids")
+								}
+
 								// pop local sgs from remote
 								diffSGs := remoteSGSet.Difference(localSGSet)
 								if len(diffSGs.List()) > 0 {
@@ -789,17 +877,20 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 				}
 			}
 		}
+	}
 
 	// Here we catch any remote rules that have not been stripped of all self,
 	// cidrs, and security groups. We'll add remote rules here that have not been
 	// matched locally, and let the graph sort things out. This will happen when
 	// rules are added externally to Terraform
 	for _, r := range remote {
-		var lenCidr, lenSGs int
+		var lenCidr, lenPrefixLists, lenSGs int
 		if rCidrs, ok := r["cidr_blocks"]; ok {
 			lenCidr = len(rCidrs.([]string))
 		}
-
+		if rPrefixLists, ok := r["prefix_list_ids"]; ok {
+			lenPrefixLists = len(rPrefixLists.([]string))
+		}
 		if rawSGs, ok := r["security_groups"]; ok {
 			lenSGs = len(rawSGs.(*schema.Set).List())
 		}
@@ -810,7 +901,7 @@ func matchRules(rType string, local []interface{}, remote []map[string]interface
 			}
 		}
 
-		if lenSGs+lenCidr > 0 {
+		if lenSGs+lenCidr+lenPrefixLists > 0 {
 			log.Printf("[DEBUG] Found a remote Rule that wasn't empty: (%#v)", r)
 			saves = append(saves, r)
 		}

builtin/providers/aws/resource_aws_security_group_rule.go

@@ -59,6 +59,13 @@ func resourceAwsSecurityGroupRule() *schema.Resource {
 				Elem:     &schema.Schema{Type: schema.TypeString},
 			},
 
+			"prefix_list_ids": &schema.Schema{
+				Type:     schema.TypeList,
+				Optional: true,
+				ForceNew: true,
+				Elem:     &schema.Schema{Type: schema.TypeString},
+			},
+
 			"security_group_id": &schema.Schema{
 				Type:     schema.TypeString,
 				Required: true,
@@ -363,6 +370,19 @@ func findRuleMatch(p *ec2.IpPermission, rules []*ec2.IpPermission, isVPC bool) *
 			continue
 		}
 
+		remaining = len(p.PrefixListIds)
+		for _, pl := range p.PrefixListIds {
+			for _, rpl := range r.PrefixListIds {
+				if *pl.PrefixListId == *rpl.PrefixListId {
+					remaining--
+				}
+			}
+		}
+
+		if remaining > 0 {
+			continue
+		}
+
 		remaining = len(p.UserIdGroupPairs)
 		for _, ip := range p.UserIdGroupPairs {
 			for _, rip := range r.UserIdGroupPairs {
@@ -413,6 +433,18 @@ func ipPermissionIDHash(sg_id, ruleType string, ip *ec2.IpPermission) string {
 		}
 	}
 
+	if len(ip.PrefixListIds) > 0 {
+		s := make([]string, len(ip.PrefixListIds))
+		for i, pl := range ip.PrefixListIds {
+			s[i] = *pl.PrefixListId
+		}
+		sort.Strings(s)
+
+		for _, v := range s {
+			buf.WriteString(fmt.Sprintf("%s-", v))
+		}
+	}
+
 	if len(ip.UserIdGroupPairs) > 0 {
 		sort.Sort(ByGroupPair(ip.UserIdGroupPairs))
 		for _, pair := range ip.UserIdGroupPairs {
@@ -494,6 +526,18 @@ func expandIPPerm(d *schema.ResourceData, sg *ec2.SecurityGroup) (*ec2.IpPermiss
 		}
 	}
 
+	if raw, ok := d.GetOk("prefix_list_ids"); ok {
+		list := raw.([]interface{})
+		perm.PrefixListIds = make([]*ec2.PrefixListId, len(list))
+		for i, v := range list {
+			prefixListID, ok := v.(string)
+			if !ok {
+				return nil, fmt.Errorf("empty element found in prefix_list_ids - consider using the compact function")
+			}
+			perm.PrefixListIds[i] = &ec2.PrefixListId{PrefixListId: aws.String(prefixListID)}
+		}
+	}
+
 	return &perm, nil
 }
 
@@ -514,6 +558,13 @@ func setFromIPPerm(d *schema.ResourceData, sg *ec2.SecurityGroup, rule *ec2.IpPe
 	// 'self' is false by default. Below, we range over the group ids and set true
 	// if the parent sg id is found
 	d.Set("self", false)
+
+	var pl []string
+	for _, p := range rule.PrefixListIds {
+		pl = append(pl, *p.PrefixListId)
+	}
+	d.Set("prefix_list_ids", pl)
+
 	if len(rule.UserIdGroupPairs) > 0 {
 		s := rule.UserIdGroupPairs[0]
 

builtin/providers/aws/resource_aws_security_group_rule_test.go

@@ -416,6 +416,25 @@ func TestAccAWSSecurityGroupRule_Race(t *testing.T) {
 	})
 }
 
+func TestAccAWSSecurityGroupRule_PrefixListEgress(t *testing.T) {
+	var group ec2.SecurityGroup
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckAWSSecurityGroupRuleDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: testAccAWSSecurityGroupRulePrefixListEgressConfig,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckAWSSecurityGroupRuleExists("aws_security_group.egress", &group),
+					testAccCheckAWSSecurityGroupRuleAttributes("aws_security_group_rule.egress_1", &group, nil, "egress"),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckAWSSecurityGroupRuleDestroy(s *terraform.State) error {
 	conn := testAccProvider.Meta().(*AWSClient).ec2conn
 
@@ -879,3 +898,52 @@ var testAccAWSSecurityGroupRuleRace = func() string {
 	}
 	return b.String()
 }()
+
+const testAccAWSSecurityGroupRulePrefixListEgressConfig = `
+
+resource "aws_vpc" "tf_sg_prefix_list_egress_test" {
+    cidr_block = "10.0.0.0/16"
+    tags {
+            Name = "tf_sg_prefix_list_egress_test"
+    }
+}
+
+resource "aws_route_table" "default" {
+    vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+}
+
+resource "aws_vpc_endpoint" "s3-us-west-2" {
+  	vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+  	service_name = "com.amazonaws.us-west-2.s3"
+  	route_table_ids = ["${aws_route_table.default.id}"]
+  	policy = <<POLICY
+{
+	"Version": "2012-10-17",
+	"Statement": [
+		{
+			"Sid":"AllowAll",
+			"Effect":"Allow",
+			"Principal":"*",
+			"Action":"*",
+			"Resource":"*"
+		}
+	]
+}
+POLICY
+}
+
+resource "aws_security_group" "egress" {
+    name = "terraform_acceptance_test_prefix_list_egress"
+    description = "Used in the terraform acceptance tests"
+    vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+}
+
+resource "aws_security_group_rule" "egress_1" {
+  type = "egress"
+  protocol = "-1"
+  from_port = 0
+  to_port = 0
+  prefix_list_ids = ["${aws_vpc_endpoint.s3-us-west-2.prefix_list_id}"]
+	security_group_id = "${aws_security_group.egress.id}"
+}
+`

builtin/providers/aws/resource_aws_security_group_test.go

@@ -174,6 +174,18 @@ func TestResourceAwsSecurityGroupIPPermGather(t *testing.T) {
 				},
 			},
 		},
+		&ec2.IpPermission{
+			IpProtocol:    aws.String("-1"),
+			FromPort:      aws.Int64(int64(0)),
+			ToPort:        aws.Int64(int64(0)),
+			PrefixListIds: []*ec2.PrefixListId{&ec2.PrefixListId{PrefixListId: aws.String("pl-12345678")}},
+			UserIdGroupPairs: []*ec2.UserIdGroupPair{
+				// VPC
+				&ec2.UserIdGroupPair{
+					GroupId: aws.String("sg-22222"),
+				},
+			},
+		},
 	}
 
 	local := []map[string]interface{}{
@@ -201,6 +213,15 @@ func TestResourceAwsSecurityGroupIPPermGather(t *testing.T) {
 				"amazon-elb/amazon-elb-sg",
 			}),
 		},
+		map[string]interface{}{
+			"protocol":        "-1",
+			"from_port":       int64(0),
+			"to_port":         int64(0),
+			"prefix_list_ids": []string{"pl-12345678"},
+			"security_groups": schema.NewSet(schema.HashString, []interface{}{
+				"sg-22222",
+			}),
+		},
 	}
 
 	out := resourceAwsSecurityGroupIPPermGather("sg-11111", raw, aws.String("12345"))
@@ -846,6 +867,27 @@ func TestAccAWSSecurityGroup_ingressWithCidrAndSGs_classic(t *testing.T) {
 	})
 }
 
+func TestAccAWSSecurityGroup_egressWithPrefixList(t *testing.T) {
+	var group ec2.SecurityGroup
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckAWSSecurityGroupDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: testAccAWSSecurityGroupConfigPrefixListEgress,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckAWSSecurityGroupExists("aws_security_group.egress", &group),
+					testAccCheckAWSSecurityGroupPrefixListAttributes(&group),
+					resource.TestCheckResourceAttr(
+						"aws_security_group.egress", "egress.#", "1"),
+				),
+			},
+		},
+	})
+}
+
 func testAccCheckAWSSecurityGroupSGandCidrAttributes(group *ec2.SecurityGroup) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		if *group.GroupName != "terraform_acceptance_test_example" {
@@ -883,6 +925,31 @@ func testAccCheckAWSSecurityGroupSGandCidrAttributes(group *ec2.SecurityGroup) r
 	}
 }
 
+func testAccCheckAWSSecurityGroupPrefixListAttributes(group *ec2.SecurityGroup) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		if *group.GroupName != "terraform_acceptance_test_prefix_list_egress" {
+			return fmt.Errorf("Bad name: %s", *group.GroupName)
+		}
+		if *group.Description != "Used in the terraform acceptance tests" {
+			return fmt.Errorf("Bad description: %s", *group.Description)
+		}
+		if len(group.IpPermissionsEgress) == 0 {
+			return fmt.Errorf("No egress IPPerms")
+		}
+		if len(group.IpPermissionsEgress) != 1 {
+			return fmt.Errorf("Expected 1 egress rule, got %d", len(group.IpPermissions))
+		}
+
+		p := group.IpPermissionsEgress[0]
+
+		if len(p.PrefixListIds) != 1 {
+			return fmt.Errorf("Expected 1 prefix list, got %d", len(p.PrefixListIds))
+		}
+
+		return nil
+	}
+}
+
 func testAccCheckAWSSecurityGroupAttributesChanged(group *ec2.SecurityGroup) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		p := []*ec2.IpPermission{
@@ -1632,3 +1699,49 @@ resource "aws_security_group_rule" "allow_all-1" {
   security_group_id = "${aws_security_group.allow_all.id}"
 }
 `
+
+const testAccAWSSecurityGroupConfigPrefixListEgress = `
+resource "aws_vpc" "tf_sg_prefix_list_egress_test" {
+    cidr_block = "10.0.0.0/16"
+    tags {
+            Name = "tf_sg_prefix_list_egress_test"
+    }
+}
+
+resource "aws_route_table" "default" {
+    vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+}
+
+resource "aws_vpc_endpoint" "s3-us-west-2" {
+  	vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+  	service_name = "com.amazonaws.us-west-2.s3"
+  	route_table_ids = ["${aws_route_table.default.id}"]
+  	policy = <<POLICY
+{
+	"Version": "2012-10-17",
+	"Statement": [
+		{
+			"Sid":"AllowAll",
+			"Effect":"Allow",
+			"Principal":"*",
+			"Action":"*",
+			"Resource":"*"
+		}
+	]
+}
+POLICY
+}
+
+resource "aws_security_group" "egress" {
+    name = "terraform_acceptance_test_prefix_list_egress"
+    description = "Used in the terraform acceptance tests"
+    vpc_id = "${aws_vpc.tf_sg_prefix_list_egress_test.id}"
+ 
+    egress {
+      protocol = "-1"
+      from_port = 0
+      to_port = 0
+      prefix_list_ids = ["${aws_vpc_endpoint.s3-us-west-2.prefix_list_id}"]
+    }
+}
+`

builtin/providers/aws/structure.go

@@ -208,6 +208,13 @@ func expandIPPerms(
 			}
 		}
 
+		if raw, ok := m["prefix_list_ids"]; ok {
+			list := raw.([]interface{})
+			for _, v := range list {
+				perm.PrefixListIds = append(perm.PrefixListIds, &ec2.PrefixListId{PrefixListId: aws.String(v.(string))})
+			}
+		}
+
 		perms[i] = &perm
 	}
 

website/source/docs/providers/aws/r/security_group.html.markdown

@@ -38,6 +38,7 @@ resource "aws_security_group" "allow_all" {
       to_port = 0
       protocol = "-1"
       cidr_blocks = ["0.0.0.0/0"]
+      prefix_list_ids = ["pl-12c4e678"]
   }
 }
 ```
@@ -96,6 +97,7 @@ The `ingress` block supports:
 The `egress` block supports:
 
 * `cidr_blocks` - (Optional) List of CIDR blocks.
+* `prefix_list_ids` - (Optional) List of prefix list IDs (for allowing access to VPC endpoints)
 * `from_port` - (Required) The start port (or ICMP type number if protocol is "icmp")
 * `protocol` - (Required) The protocol. If you select a protocol of
 "-1", you must specify a "from_port" and "to_port" equal to 0.
@@ -119,6 +121,26 @@ be in place, you can use this `egress` block:
       cidr_blocks = ["0.0.0.0/0"]
     }
 
+## Usage with prefix list IDs
+
+Prefix list IDs are manged by AWS internally. Prefix list IDs
+are associated with a prefix list name, or service name, that is linked to a specific region.
+Prefix list IDs are exported on VPC Endpoints, so you can use this format:
+
+```
+    ...
+      egress {
+        from_port = 0
+        to_port = 0
+        protocol = "-1"
+        prefix_list_ids = ["${aws_vpc_endpoint.my_endpoint.prefix_list_id}"]
+      }
+    ...
+    resource "aws_vpc_endpoint" "my_endpoint" {
+      ...
+    }
+```
+
 ## Attributes Reference
 
 The following attributes are exported:

website/source/docs/providers/aws/r/security_group_rule.html.markdown

@@ -29,6 +29,7 @@ resource "aws_security_group_rule" "allow_all" {
     to_port = 65535
     protocol = "tcp"
     cidr_blocks = ["0.0.0.0/0"]
+    prefix_list_ids = ["pl-12c4e678"]
 
     security_group_id = "sg-123456"
 }
@@ -41,6 +42,8 @@ The following arguments are supported:
 * `type` - (Required) The type of rule being created. Valid options are `ingress` (inbound)
 or `egress` (outbound).
 * `cidr_blocks` - (Optional) List of CIDR blocks. Cannot be specified with `source_security_group_id`.
+* `prefix_list_ids` - (Optional) List of prefix list IDs (for allowing access to VPC endpoints).
+Only valid with `egress`.
 * `from_port` - (Required) The start port (or ICMP type number if protocol is "icmp").
 * `protocol` - (Required) The protocol.
 * `security_group_id` - (Required) The security group to apply this rule to.
@@ -50,6 +53,27 @@ or `egress` (outbound).
      a source to this ingress rule.
 * `to_port` - (Required) The end range port.
 
+## Usage with prefix list IDs
+
+Prefix list IDs are manged by AWS internally. Prefix list IDs
+are associated with a prefix list name, or service name, that is linked to a specific region.
+Prefix list IDs are exported on VPC Endpoints, so you can use this format:
+
+```
+resource "aws_security_group_rule" "allow_all" {
+    type = "egress"
+    to_port = 0
+    protocol = "-1"
+    prefix_list_ids = ["${aws_vpc_endpoint.my_endpoint.prefix_list_id}"]
+    from_port = 0
+    security_group_id = "sg-123456"
+}
+...
+resource "aws_vpc_endpoint" "my_endpoint" {
+  ...
+}
+```
+
 ## Attributes Reference
 
 The following attributes are exported: