provider/aws: Refresh `aws_autoscaling_policy` from state on 404 (#8430)

* provider/aws: Refresh `aws_autoscaling_policy` from state on 404

Fixes #8386

When an Autoscaling Group Or an Autoscaling Group Policy has been
deleted manually, terraform was throwing an error as follows:

```
* aws_autoscaling_policy.increase: Error retrieving scaling policies: ValidationError: Group sandbox-logs-logstash-wxhsckky3ndpzd7b3kmyontngy not found
        status code: 400, request id: 56a89814-6884-11e6-b3a8-d364cf04223b
```

We now refresh from state on a ValidationError - this is a common 4xx error according to AWS documentation http://docs.aws.amazon.com/AutoScaling/latest/APIReference/CommonErrors.html

```
%make testacc TEST=./builtin/providers/aws TESTARGS='-run=TestAccAWSAutoscalingPolicy_disappears'
==> Checking that code complies with gofmt requirements...
/Users/stacko/Code/go/bin/stringer
go generate $(go list ./... | grep -v /terraform/vendor/)
TF_ACC=1 go test ./builtin/providers/aws -v -run=TestAccAWSAutoscalingPolicy_disappears -timeout 120m
=== RUN   TestAccAWSAutoscalingPolicy_disappears
--- PASS: TestAccAWSAutoscalingPolicy_disappears (203.61s)
PASS
ok  	github.com/hashicorp/terraform/builtin/providers/aws	203.633s
```

* Update resource_aws_autoscaling_policy.go

builtin/providers/aws/resource_aws_autoscaling_policy.go

@@ -6,6 +6,7 @@ import (
 	"log"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/hashicorp/terraform/helper/hashcode"
 	"github.com/hashicorp/terraform/helper/schema"
@@ -271,6 +272,13 @@ func getAwsAutoscalingPolicy(d *schema.ResourceData, meta interface{}) (*autosca
 	log.Printf("[DEBUG] AutoScaling Scaling Policy Describe Params: %#v", params)
 	resp, err := autoscalingconn.DescribePolicies(&params)
 	if err != nil {
+		//A ValidationError here can mean that either the Policy is missing OR the Autoscaling Group is missing
+		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "ValidationError" {
+			log.Printf("[WARNING] %s not found, removing from state", d.Id())
+			d.SetId("")
+
+			return nil, nil
+		}
 		return nil, fmt.Errorf("Error retrieving scaling policies: %s", err)
 	}
 

builtin/providers/aws/resource_aws_autoscaling_policy_test.go

@@ -2,10 +2,14 @@ package aws
 
 import (
 	"fmt"
+	"log"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/terraform/helper/acctest"
 	"github.com/hashicorp/terraform/helper/resource"
 	"github.com/hashicorp/terraform/terraform"
@@ -44,6 +48,66 @@ func TestAccAWSAutoscalingPolicy_basic(t *testing.T) {
 	})
 }
 
+func TestAccAWSAutoscalingPolicy_disappears(t *testing.T) {
+	var policy autoscaling.ScalingPolicy
+
+	name := fmt.Sprintf("terraform-test-foobar-%s", acctest.RandString(5))
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckAWSAutoscalingPolicyDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: testAccAWSAutoscalingPolicyConfig(name),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckScalingPolicyExists("aws_autoscaling_policy.foobar_simple", &policy),
+					testAccCheckScalingPolicyDisappears(&policy),
+				),
+				ExpectNonEmptyPlan: true,
+			},
+		},
+	})
+}
+
+func testAccCheckScalingPolicyDisappears(conf *autoscaling.ScalingPolicy) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		conn := testAccProvider.Meta().(*AWSClient).autoscalingconn
+
+		params := &autoscaling.DeletePolicyInput{
+			AutoScalingGroupName: conf.AutoScalingGroupName,
+			PolicyName:           conf.PolicyName,
+		}
+
+		log.Printf("TEST %s", spew.Sdump(params))
+		_, err := conn.DeletePolicy(params)
+		if err != nil {
+			return err
+		}
+
+		return resource.Retry(10*time.Minute, func() *resource.RetryError {
+			params := &autoscaling.DescribePoliciesInput{
+				AutoScalingGroupName: conf.AutoScalingGroupName,
+				PolicyNames:          []*string{conf.PolicyName},
+			}
+			resp, err := conn.DescribePolicies(params)
+			if err != nil {
+				cgw, ok := err.(awserr.Error)
+				if ok && cgw.Code() == "ValidationError" {
+					return nil
+				}
+				return resource.NonRetryableError(
+					fmt.Errorf("Error retrieving Autoscaling Policy: %s", err))
+			}
+			if resp.ScalingPolicies == nil || len(resp.ScalingPolicies) == 0 {
+				return nil
+			}
+			return resource.RetryableError(fmt.Errorf(
+				"Waiting for Autoscaling Policy: %v", conf.PolicyName))
+		})
+	}
+}
+
 func TestAccAWSAutoscalingPolicy_upgrade(t *testing.T) {
 	var policy autoscaling.ScalingPolicy
 
@@ -96,6 +160,8 @@ func testAccCheckScalingPolicyExists(n string, policy *autoscaling.ScalingPolicy
 			return fmt.Errorf("ScalingPolicy not found")
 		}
 
+		*policy = *resp.ScalingPolicies[0]
+
 		return nil
 	}
 }