From 063454e9b80b58830454b7e50fad7406bcfe3ee7 Mon Sep 17 00:00:00 2001 From: Paul Hinze Date: Wed, 6 May 2015 18:34:20 -0500 Subject: [PATCH] provider/aws: wait for ASG capacity on creation On ASG creation, waits for up to 10m for desired_capacity or min_size healthy nodes to show up in the group before continuing. With CBD and proper HealthCheck tuning, this allows us guarantee safe ASG replacement. --- .../aws/resource_aws_autoscaling_group.go | 52 +++++++++++++++++-- .../resource_aws_autoscaling_group_test.go | 22 ++++++++ .../providers/aws/r/autoscale.html.markdown | 11 +++- 3 files changed, 80 insertions(+), 5 deletions(-) diff --git a/builtin/providers/aws/resource_aws_autoscaling_group.go b/builtin/providers/aws/resource_aws_autoscaling_group.go index 051c54c04..6df767d14 100644 --- a/builtin/providers/aws/resource_aws_autoscaling_group.go +++ b/builtin/providers/aws/resource_aws_autoscaling_group.go @@ -174,6 +174,10 @@ func resourceAwsAutoscalingGroupCreate(d *schema.ResourceData, meta interface{}) d.SetId(d.Get("name").(string)) log.Printf("[INFO] AutoScaling Group ID: %s", d.Id()) + if err := waitForASGCapacity(d, meta); err != nil { + return err + } + return resourceAwsAutoscalingGroupRead(d, meta) } @@ -225,10 +229,10 @@ func resourceAwsAutoscalingGroupUpdate(d *schema.ResourceData, meta interface{}) if d.HasChange("max_size") { opts.MaxSize = aws.Long(int64(d.Get("max_size").(int))) } - + if d.HasChange("health_check_grace_period") { - opts.HealthCheckGracePeriod = aws.Long(int64(d.Get("health_check_grace_period").(int))) - } + opts.HealthCheckGracePeriod = aws.Long(int64(d.Get("health_check_grace_period").(int))) + } if err := setAutoscalingTags(autoscalingconn, d); err != nil { return err @@ -359,3 +363,45 @@ func resourceAwsAutoscalingGroupDrain(d *schema.ResourceData, meta interface{}) return fmt.Errorf("group still has %d instances", len(g.Instances)) }) } + +var waitForASGCapacityTimeout = 10 * time.Minute + +// Waits for a minimum number of healthy instances to show up as healthy in the +// ASG before continuing. Waits up to `waitForASGCapacityTimeout` for +// "desired_capacity", or "min_size" if desired capacity is not specified. +func waitForASGCapacity(d *schema.ResourceData, meta interface{}) error { + waitFor := d.Get("min_size").(int) + if v := d.Get("desired_capacity").(int); v > 0 { + waitFor = v + } + + log.Printf("[DEBUG] Waiting for group to have %d healthy instances", waitFor) + return resource.Retry(waitForASGCapacityTimeout, func() error { + g, err := getAwsAutoscalingGroup(d, meta) + if err != nil { + return resource.RetryError{Err: err} + } + if g == nil { + return nil + } + + healthy := 0 + for _, i := range g.Instances { + if i.HealthStatus == nil { + continue + } + if strings.EqualFold(*i.HealthStatus, "Healthy") { + healthy++ + } + } + + log.Printf( + "[DEBUG] %q has %d/%d healthy instances", d.Id(), healthy, waitFor) + + if healthy >= waitFor { + return nil + } + + return fmt.Errorf("Waiting for healthy instances: %d/%d", healthy, waitFor) + }) +} diff --git a/builtin/providers/aws/resource_aws_autoscaling_group_test.go b/builtin/providers/aws/resource_aws_autoscaling_group_test.go index 46b43cdf6..cbb4ae514 100644 --- a/builtin/providers/aws/resource_aws_autoscaling_group_test.go +++ b/builtin/providers/aws/resource_aws_autoscaling_group_test.go @@ -3,6 +3,7 @@ package aws import ( "fmt" "reflect" + "strings" "testing" "github.com/awslabs/aws-sdk-go/aws" @@ -24,6 +25,7 @@ func TestAccAWSAutoScalingGroup_basic(t *testing.T) { Config: testAccAWSAutoScalingGroupConfig, Check: resource.ComposeTestCheckFunc( testAccCheckAWSAutoScalingGroupExists("aws_autoscaling_group.bar", &group), + testAccCheckAWSAutoScalingGroupHealthyCapacity(&group, 2), testAccCheckAWSAutoScalingGroupAttributes(&group), resource.TestCheckResourceAttr( "aws_autoscaling_group.bar", "availability_zones.2487133097", "us-west-2a"), @@ -116,6 +118,7 @@ func TestAccAWSAutoScalingGroup_WithLoadBalancer(t *testing.T) { }, }) } + func testAccCheckAWSAutoScalingGroupDestroy(s *terraform.State) error { conn := testAccProvider.Meta().(*AWSClient).autoscalingconn @@ -261,6 +264,25 @@ func testLaunchConfigurationName(n string, lc *autoscaling.LaunchConfiguration) } } +func testAccCheckAWSAutoScalingGroupHealthyCapacity( + g *autoscaling.AutoScalingGroup, exp int) resource.TestCheckFunc { + return func(s *terraform.State) error { + healthy := 0 + for _, i := range g.Instances { + if i.HealthStatus == nil { + continue + } + if strings.EqualFold(*i.HealthStatus, "Healthy") { + healthy++ + } + } + if healthy < exp { + return fmt.Errorf("Expected at least %d healthy, got %d.", exp, healthy) + } + return nil + } +} + const testAccAWSAutoScalingGroupConfig = ` resource "aws_launch_configuration" "foobar" { name = "foobarautoscaling-terraform-test" diff --git a/website/source/docs/providers/aws/r/autoscale.html.markdown b/website/source/docs/providers/aws/r/autoscale.html.markdown index fe0643b72..7f3a4be9d 100644 --- a/website/source/docs/providers/aws/r/autoscale.html.markdown +++ b/website/source/docs/providers/aws/r/autoscale.html.markdown @@ -43,12 +43,19 @@ The following arguments are supported: * `name` - (Required) The name of the auto scale group. * `max_size` - (Required) The maximum size of the auto scale group. -* `min_size` - (Required) The minimum size of the auto scale group. +* `min_size` - (Required) The minimum size of the auto scale group. Terraform + waits after ASG creation for this number of healthy instances to show up in + the ASG before continuing. Currently, it will wait for a maxiumum of 10m, if + ASG creation is taking more than a few minutes, it's worth investigating for + scaling actvity errors caused by problems with the selected Launch + Configuration. * `availability_zones` - (Required) A list of AZs to launch resources in. * `launch_configuration` - (Required) The ID of the launch configuration to use. * `health_check_grace_period` - (Optional) Time after instance comes into service before checking health. * `health_check_type` - (Optional) "EC2" or "ELB". Controls how health checking is done. -* `desired_capacity` - (Optional) The number of Amazon EC2 instances that should be running in the group. +* `desired_capacity` - (Optional) The number of Amazon EC2 instances that + should be running in the group. (If this is specified, Terraform will wait for + this number of healthy instances after ASG creation instead of `min_size`.) * `force_delete` - (Optional) Allows deleting the autoscaling group without waiting for all instances in the pool to terminate. * `load_balancers` (Optional) A list of load balancer names to add to the autoscaling