provider/aws: wait for ASG capacity on creation

On ASG creation, waits for up to 10m for desired_capacity or min_size
healthy nodes to show up in the group before continuing.

With CBD and proper HealthCheck tuning, this allows us guarantee safe
ASG replacement.

builtin/providers/aws/resource_aws_autoscaling_group.go

@@ -174,6 +174,10 @@ func resourceAwsAutoscalingGroupCreate(d *schema.ResourceData, meta interface{})
 	d.SetId(d.Get("name").(string))
 	log.Printf("[INFO] AutoScaling Group ID: %s", d.Id())
 
+	if err := waitForASGCapacity(d, meta); err != nil {
+		return err
+	}
+
 	return resourceAwsAutoscalingGroupRead(d, meta)
 }
 
@@ -359,3 +363,45 @@ func resourceAwsAutoscalingGroupDrain(d *schema.ResourceData, meta interface{})
 		return fmt.Errorf("group still has %d instances", len(g.Instances))
 	})
 }
+
+var waitForASGCapacityTimeout = 10 * time.Minute
+
+// Waits for a minimum number of healthy instances to show up as healthy in the
+// ASG before continuing. Waits up to `waitForASGCapacityTimeout` for
+// "desired_capacity", or "min_size" if desired capacity is not specified.
+func waitForASGCapacity(d *schema.ResourceData, meta interface{}) error {
+	waitFor := d.Get("min_size").(int)
+	if v := d.Get("desired_capacity").(int); v > 0 {
+		waitFor = v
+	}
+
+	log.Printf("[DEBUG] Waiting for group to have %d healthy instances", waitFor)
+	return resource.Retry(waitForASGCapacityTimeout, func() error {
+		g, err := getAwsAutoscalingGroup(d, meta)
+		if err != nil {
+			return resource.RetryError{Err: err}
+		}
+		if g == nil {
+			return nil
+		}
+
+		healthy := 0
+		for _, i := range g.Instances {
+			if i.HealthStatus == nil {
+				continue
+			}
+			if strings.EqualFold(*i.HealthStatus, "Healthy") {
+				healthy++
+			}
+		}
+
+		log.Printf(
+			"[DEBUG] %q has %d/%d healthy instances", d.Id(), healthy, waitFor)
+
+		if healthy >= waitFor {
+			return nil
+		}
+
+		return fmt.Errorf("Waiting for healthy instances: %d/%d", healthy, waitFor)
+	})
+}

builtin/providers/aws/resource_aws_autoscaling_group_test.go

@@ -3,6 +3,7 @@ package aws
 import (
 	"fmt"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/awslabs/aws-sdk-go/aws"
@@ -24,6 +25,7 @@ func TestAccAWSAutoScalingGroup_basic(t *testing.T) {
 				Config: testAccAWSAutoScalingGroupConfig,
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckAWSAutoScalingGroupExists("aws_autoscaling_group.bar", &group),
+					testAccCheckAWSAutoScalingGroupHealthyCapacity(&group, 2),
 					testAccCheckAWSAutoScalingGroupAttributes(&group),
 					resource.TestCheckResourceAttr(
 						"aws_autoscaling_group.bar", "availability_zones.2487133097", "us-west-2a"),
@@ -116,6 +118,7 @@ func TestAccAWSAutoScalingGroup_WithLoadBalancer(t *testing.T) {
 		},
 	})
 }
+
 func testAccCheckAWSAutoScalingGroupDestroy(s *terraform.State) error {
 	conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
 
@@ -261,6 +264,25 @@ func testLaunchConfigurationName(n string, lc *autoscaling.LaunchConfiguration)
 	}
 }
 
+func testAccCheckAWSAutoScalingGroupHealthyCapacity(
+	g *autoscaling.AutoScalingGroup, exp int) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		healthy := 0
+		for _, i := range g.Instances {
+			if i.HealthStatus == nil {
+				continue
+			}
+			if strings.EqualFold(*i.HealthStatus, "Healthy") {
+				healthy++
+			}
+		}
+		if healthy < exp {
+			return fmt.Errorf("Expected at least %d healthy, got %d.", exp, healthy)
+		}
+		return nil
+	}
+}
+
 const testAccAWSAutoScalingGroupConfig = `
 resource "aws_launch_configuration" "foobar" {
   name = "foobarautoscaling-terraform-test"

website/source/docs/providers/aws/r/autoscale.html.markdown

@@ -43,12 +43,19 @@ The following arguments are supported:
 
 * `name` - (Required) The name of the auto scale group.
 * `max_size` - (Required) The maximum size of the auto scale group.
-* `min_size` - (Required) The minimum size of the auto scale group.
+* `min_size` - (Required) The minimum size of the auto scale group. Terraform
+  waits after ASG creation for this number of healthy instances to show up in
+  the ASG before continuing. Currently, it will wait for a maxiumum of 10m, if
+  ASG creation is taking more than a few minutes, it's worth investigating for
+  scaling actvity errors caused by problems with the selected Launch
+  Configuration.
 * `availability_zones` - (Required) A list of AZs to launch resources in.
 * `launch_configuration` - (Required) The ID of the launch configuration to use.
 * `health_check_grace_period` - (Optional) Time after instance comes into service before checking health.
 * `health_check_type` - (Optional) "EC2" or "ELB". Controls how health checking is done.
-* `desired_capacity` - (Optional) The number of Amazon EC2 instances that should be running in the group.
+* `desired_capacity` - (Optional) The number of Amazon EC2 instances that
+  should be running in the group. (If this is specified, Terraform will wait for
+  this number of healthy instances after ASG creation instead of `min_size`.)
 * `force_delete` - (Optional) Allows deleting the autoscaling group without waiting
    for all instances in the pool to terminate.
 * `load_balancers` (Optional) A list of load balancer names to add to the autoscaling