From 2b9f4f895eae1d58972b106ebafeb5515fc2603f Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 1 Oct 2015 15:12:46 -0700 Subject: [PATCH 1/3] provider/aws: Add support for aws_elasticsearch_domain --- builtin/providers/aws/config.go | 5 + builtin/providers/aws/provider.go | 1 + .../aws/resource_aws_elasticsearch_domain.go | 399 ++++++++++++++++++ builtin/providers/aws/structure.go | 111 +++++ 4 files changed, 516 insertions(+) create mode 100644 builtin/providers/aws/resource_aws_elasticsearch_domain.go diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index a20405997..badc3e20e 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -18,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/efs" "github.com/aws/aws-sdk-go/service/elasticache" + elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/kinesis" @@ -51,6 +52,7 @@ type AWSClient struct { ecsconn *ecs.ECS efsconn *efs.EFS elbconn *elb.ELB + esconn *elasticsearch.ElasticsearchService autoscalingconn *autoscaling.AutoScaling s3conn *s3.S3 sqsconn *sqs.SQS @@ -157,6 +159,9 @@ func (c *Config) Client() (interface{}, error) { log.Println("[INFO] Initializing EFS Connection") client.efsconn = efs.New(awsConfig) + log.Println("[INFO] Initializing ElasticSearch Connection") + client.esconn = elasticsearch.New(awsConfig) + log.Println("[INFO] Initializing Route 53 connection") client.r53conn = route53.New(usEast1AwsConfig) diff --git a/builtin/providers/aws/provider.go b/builtin/providers/aws/provider.go index c915c61fb..42b1d6242 100644 --- a/builtin/providers/aws/provider.go +++ b/builtin/providers/aws/provider.go @@ -182,6 +182,7 @@ func Provider() terraform.ResourceProvider { "aws_elasticache_parameter_group": resourceAwsElasticacheParameterGroup(), "aws_elasticache_security_group": resourceAwsElasticacheSecurityGroup(), "aws_elasticache_subnet_group": resourceAwsElasticacheSubnetGroup(), + "aws_elasticsearch_domain": resourceAwsElasticSearchDomain(), "aws_elb": resourceAwsElb(), "aws_flow_log": resourceAwsFlowLog(), "aws_iam_access_key": resourceAwsIamAccessKey(), diff --git a/builtin/providers/aws/resource_aws_elasticsearch_domain.go b/builtin/providers/aws/resource_aws_elasticsearch_domain.go new file mode 100644 index 000000000..8f2d6c9c9 --- /dev/null +++ b/builtin/providers/aws/resource_aws_elasticsearch_domain.go @@ -0,0 +1,399 @@ +package aws + +import ( + "fmt" + "log" + "regexp" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceAwsElasticSearchDomain() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsElasticSearchDomainCreate, + Read: resourceAwsElasticSearchDomainRead, + Update: resourceAwsElasticSearchDomainUpdate, + Delete: resourceAwsElasticSearchDomainDelete, + + Schema: map[string]*schema.Schema{ + "access_policies": &schema.Schema{ + Type: schema.TypeString, + StateFunc: normalizeJson, + Optional: true, + }, + "advanced_options": &schema.Schema{ + Type: schema.TypeMap, + Optional: true, + Computed: true, + }, + "domain_name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + if !regexp.MustCompile(`^[0-9A-Za-z]+`).MatchString(value) { + errors = append(errors, fmt.Errorf( + "%q must start with a letter or number", k)) + } + if !regexp.MustCompile(`^[0-9A-Za-z][0-9a-z-]+$`).MatchString(value) { + errors = append(errors, fmt.Errorf( + "%q can only contain lowercase characters, numbers and hyphens", k)) + } + return + }, + }, + "arn": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "domain_id": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "endpoint": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "ebs_options": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "ebs_enabled": &schema.Schema{ + Type: schema.TypeBool, + Required: true, + }, + "iops": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + }, + "volume_size": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + }, + "volume_type": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "cluster_config": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "dedicated_master_count": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + }, + "dedicated_master_enabled": &schema.Schema{ + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "dedicated_master_type": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + "instance_count": &schema.Schema{ + Type: schema.TypeInt, + Optional: true, + Default: 1, + }, + "instance_type": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Default: "m3.medium.elasticsearch", + }, + "zone_awareness_enabled": &schema.Schema{ + Type: schema.TypeBool, + Optional: true, + }, + }, + }, + }, + "snapshot_options": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "automated_snapshot_start_hour": &schema.Schema{ + Type: schema.TypeInt, + Required: true, + }, + }, + }, + }, + }, + } +} + +func resourceAwsElasticSearchDomainCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).esconn + + input := elasticsearch.CreateElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + } + + if v, ok := d.GetOk("access_policies"); ok { + input.AccessPolicies = aws.String(v.(string)) + } + + if v, ok := d.GetOk("advanced_options"); ok { + input.AdvancedOptions = stringMapToPointers(v.(map[string]interface{})) + } + + if v, ok := d.GetOk("ebs_options"); ok { + options := v.([]interface{}) + + if len(options) > 1 { + return fmt.Errorf("Only a single ebs_options block is expected") + } else if len(options) == 1 { + if options[0] == nil { + return fmt.Errorf("At least one field is expected inside ebs_options") + } + + s := options[0].(map[string]interface{}) + input.EBSOptions = expandESEBSOptions(s) + } + } + + if v, ok := d.GetOk("cluster_config"); ok { + config := v.([]interface{}) + + if len(config) > 1 { + return fmt.Errorf("Only a single cluster_config block is expected") + } else if len(config) == 1 { + if config[0] == nil { + return fmt.Errorf("At least one field is expected inside cluster_config") + } + m := config[0].(map[string]interface{}) + input.ElasticsearchClusterConfig = expandESClusterConfig(m) + } + } + + if v, ok := d.GetOk("snapshot_options"); ok { + options := v.([]interface{}) + + if len(options) > 1 { + return fmt.Errorf("Only a single snapshot_options block is expected") + } else if len(options) == 1 { + if options[0] == nil { + return fmt.Errorf("At least one field is expected inside snapshot_options") + } + + o := options[0].(map[string]interface{}) + + snapshotOptions := elasticsearch.SnapshotOptions{ + AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))), + } + + input.SnapshotOptions = &snapshotOptions + } + } + + log.Printf("[DEBUG] Creating ElasticSearch domain: %s", input) + out, err := conn.CreateElasticsearchDomain(&input) + if err != nil { + return err + } + + d.SetId(*out.DomainStatus.ARN) + + log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be created", d.Id()) + err = resource.Retry(15*time.Minute, func() error { + out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + }) + if err != nil { + return resource.RetryError{Err: err} + } + + if !*out.DomainStatus.Processing && out.DomainStatus.Endpoint != nil { + return nil + } + + return fmt.Errorf("%q: Timeout while waiting for the domain to be created", d.Id()) + }) + if err != nil { + return err + } + + log.Printf("[DEBUG] ElasticSearch domain %q created", d.Id()) + + return resourceAwsElasticSearchDomainRead(d, meta) +} + +func resourceAwsElasticSearchDomainRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).esconn + + out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + }) + if err != nil { + return err + } + + log.Printf("[DEBUG] Received ElasticSearch domain: %s", out) + + ds := out.DomainStatus + + d.Set("access_policies", *ds.AccessPolicies) + err = d.Set("advanced_options", pointersMapToStringList(ds.AdvancedOptions)) + if err != nil { + return err + } + d.Set("domain_id", *ds.DomainId) + d.Set("domain_name", *ds.DomainName) + if ds.Endpoint != nil { + d.Set("endpoint", *ds.Endpoint) + } + + err = d.Set("ebs_options", flattenESEBSOptions(ds.EBSOptions)) + if err != nil { + return err + } + err = d.Set("cluster_config", flattenESClusterConfig(ds.ElasticsearchClusterConfig)) + if err != nil { + return err + } + if ds.SnapshotOptions != nil { + d.Set("snapshot_options", map[string]interface{}{ + "automated_snapshot_start_hour": *ds.SnapshotOptions.AutomatedSnapshotStartHour, + }) + } + + d.Set("arn", *ds.ARN) + + return nil +} + +func resourceAwsElasticSearchDomainUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).esconn + + input := elasticsearch.UpdateElasticsearchDomainConfigInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + } + + if d.HasChange("access_policies") { + input.AccessPolicies = aws.String(d.Get("access_policies").(string)) + } + + if d.HasChange("advanced_options") { + input.AdvancedOptions = stringMapToPointers(d.Get("advanced_options").(map[string]interface{})) + } + + if d.HasChange("ebs_options") { + options := d.Get("ebs_options").([]interface{}) + + if len(options) > 1 { + return fmt.Errorf("Only a single ebs_options block is expected") + } else if len(options) == 1 { + s := options[0].(map[string]interface{}) + input.EBSOptions = expandESEBSOptions(s) + } + } + + if d.HasChange("cluster_config") { + config := d.Get("cluster_config").([]interface{}) + + if len(config) > 1 { + return fmt.Errorf("Only a single cluster_config block is expected") + } else if len(config) == 1 { + m := config[0].(map[string]interface{}) + input.ElasticsearchClusterConfig = expandESClusterConfig(m) + } + } + + if d.HasChange("snapshot_options") { + options := d.Get("snapshot_options").([]interface{}) + + if len(options) > 1 { + return fmt.Errorf("Only a single snapshot_options block is expected") + } else if len(options) == 1 { + o := options[0].(map[string]interface{}) + + snapshotOptions := elasticsearch.SnapshotOptions{ + AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))), + } + + input.SnapshotOptions = &snapshotOptions + } + } + + _, err := conn.UpdateElasticsearchDomainConfig(&input) + if err != nil { + return err + } + + err = resource.Retry(25*time.Minute, func() error { + out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + }) + if err != nil { + return resource.RetryError{Err: err} + } + + if *out.DomainStatus.Processing == false { + return nil + } + + return fmt.Errorf("%q: Timeout while waiting for changes to be processed", d.Id()) + }) + if err != nil { + return err + } + + return resourceAwsElasticSearchDomainRead(d, meta) +} + +func resourceAwsElasticSearchDomainDelete(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).esconn + + log.Printf("[DEBUG] Deleting ElasticSearch domain: %q", d.Get("domain_name").(string)) + _, err := conn.DeleteElasticsearchDomain(&elasticsearch.DeleteElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + }) + if err != nil { + return err + } + + log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be deleted", d.Get("domain_name").(string)) + err = resource.Retry(15*time.Minute, func() error { + out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(d.Get("domain_name").(string)), + }) + + if err != nil { + awsErr, ok := err.(awserr.Error) + if !ok { + return resource.RetryError{Err: err} + } + + if awsErr.Code() == "ResourceNotFoundException" { + return nil + } + + return resource.RetryError{Err: awsErr} + } + + if !*out.DomainStatus.Processing { + return nil + } + + return fmt.Errorf("%q: Timeout while waiting for the domain to be deleted", d.Id()) + }) + + d.SetId("") + + return err +} diff --git a/builtin/providers/aws/structure.go b/builtin/providers/aws/structure.go index dc7b6d89b..b738027f8 100644 --- a/builtin/providers/aws/structure.go +++ b/builtin/providers/aws/structure.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ecs" "github.com/aws/aws-sdk-go/service/elasticache" + elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/rds" "github.com/aws/aws-sdk-go/service/route53" @@ -479,3 +480,113 @@ func validateRdsId(v interface{}, k string) (ws []string, errors []error) { } return } + +func expandESClusterConfig(m map[string]interface{}) *elasticsearch.ElasticsearchClusterConfig { + config := elasticsearch.ElasticsearchClusterConfig{} + + if v, ok := m["dedicated_master_enabled"]; ok { + isEnabled := v.(bool) + config.DedicatedMasterEnabled = aws.Bool(isEnabled) + + if isEnabled { + if v, ok := m["dedicated_master_count"]; ok && v.(int) > 0 { + config.DedicatedMasterCount = aws.Int64(int64(v.(int))) + } + if v, ok := m["dedicated_master_type"]; ok && v.(string) != "" { + config.DedicatedMasterType = aws.String(v.(string)) + } + } + } + + if v, ok := m["instance_count"]; ok { + config.InstanceCount = aws.Int64(int64(v.(int))) + } + if v, ok := m["instance_type"]; ok { + config.InstanceType = aws.String(v.(string)) + } + + if v, ok := m["zone_awareness_enabled"]; ok { + config.ZoneAwarenessEnabled = aws.Bool(v.(bool)) + } + + return &config +} + +func flattenESClusterConfig(c *elasticsearch.ElasticsearchClusterConfig) []map[string]interface{} { + m := map[string]interface{}{} + + if c.DedicatedMasterCount != nil { + m["dedicated_master_count"] = *c.DedicatedMasterCount + } + if c.DedicatedMasterEnabled != nil { + m["dedicated_master_enabled"] = *c.DedicatedMasterEnabled + } + if c.DedicatedMasterType != nil { + m["dedicated_master_type"] = *c.DedicatedMasterType + } + if c.InstanceCount != nil { + m["instance_count"] = *c.InstanceCount + } + if c.InstanceType != nil { + m["instance_type"] = *c.InstanceType + } + if c.ZoneAwarenessEnabled != nil { + m["zone_awareness_enabled"] = *c.ZoneAwarenessEnabled + } + + return []map[string]interface{}{m} +} + +func flattenESEBSOptions(o *elasticsearch.EBSOptions) []map[string]interface{} { + m := map[string]interface{}{} + + if o.EBSEnabled != nil { + m["ebs_enabled"] = *o.EBSEnabled + } + if o.Iops != nil { + m["iops"] = *o.Iops + } + if o.VolumeSize != nil { + m["volume_size"] = *o.VolumeSize + } + if o.VolumeType != nil { + m["volume_type"] = *o.VolumeType + } + + return []map[string]interface{}{m} +} + +func expandESEBSOptions(m map[string]interface{}) *elasticsearch.EBSOptions { + options := elasticsearch.EBSOptions{} + + if v, ok := m["ebs_enabled"]; ok { + options.EBSEnabled = aws.Bool(v.(bool)) + } + if v, ok := m["iops"]; ok && v.(int) > 0 { + options.Iops = aws.Int64(int64(v.(int))) + } + if v, ok := m["volume_size"]; ok && v.(int) > 0 { + options.VolumeSize = aws.Int64(int64(v.(int))) + } + if v, ok := m["volume_type"]; ok && v.(string) != "" { + options.VolumeType = aws.String(v.(string)) + } + + return &options +} + +func pointersMapToStringList(pointers map[string]*string) map[string]interface{} { + list := make(map[string]interface{}, len(pointers)) + for i, v := range pointers { + list[i] = *v + } + return list +} + +func stringMapToPointers(m map[string]interface{}) map[string]*string { + list := make(map[string]*string, len(m)) + for i, v := range m { + list[i] = aws.String(v.(string)) + } + return list +} From c221da9aeb8d541e941a68fdc0400b8f88ecefaf Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 1 Oct 2015 15:13:12 -0700 Subject: [PATCH 2/3] provider/aws: Add acceptance test for aws_elasticsearch_domain --- .../resource_aws_elasticsearch_domain_test.go | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 builtin/providers/aws/resource_aws_elasticsearch_domain_test.go diff --git a/builtin/providers/aws/resource_aws_elasticsearch_domain_test.go b/builtin/providers/aws/resource_aws_elasticsearch_domain_test.go new file mode 100644 index 000000000..dee675d0d --- /dev/null +++ b/builtin/providers/aws/resource_aws_elasticsearch_domain_test.go @@ -0,0 +1,122 @@ +package aws + +import ( + "fmt" + "testing" + + "github.com/aws/aws-sdk-go/aws" + elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSElasticSearchDomain_basic(t *testing.T) { + var domain elasticsearch.ElasticsearchDomainStatus + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckESDomainDestroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccESDomainConfig_basic, + Check: resource.ComposeTestCheckFunc( + testAccCheckESDomainExists("aws_elasticsearch_domain.example", &domain), + ), + }, + }, + }) +} + +func TestAccAWSElasticSearchDomain_complex(t *testing.T) { + var domain elasticsearch.ElasticsearchDomainStatus + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckESDomainDestroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccESDomainConfig_complex, + Check: resource.ComposeTestCheckFunc( + testAccCheckESDomainExists("aws_elasticsearch_domain.example", &domain), + ), + }, + }, + }) +} + +func testAccCheckESDomainExists(n string, domain *elasticsearch.ElasticsearchDomainStatus) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No ES Domain ID is set") + } + + conn := testAccProvider.Meta().(*AWSClient).esconn + opts := &elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(rs.Primary.Attributes["domain_name"]), + } + + resp, err := conn.DescribeElasticsearchDomain(opts) + if err != nil { + return fmt.Errorf("Error describing domain: %s", err.Error()) + } + + *domain = *resp.DomainStatus + + return nil + } +} + +func testAccCheckESDomainDestroy(s *terraform.State) error { + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_elasticsearch_domain" { + continue + } + + conn := testAccProvider.Meta().(*AWSClient).esconn + opts := &elasticsearch.DescribeElasticsearchDomainInput{ + DomainName: aws.String(rs.Primary.Attributes["domain_name"]), + } + + _, err := conn.DescribeElasticsearchDomain(opts) + if err != nil { + return fmt.Errorf("Error describing ES domains: %q", err.Error()) + } + } + return nil +} + +const testAccESDomainConfig_basic = ` +resource "aws_elasticsearch_domain" "example" { + domain_name = "tf-test-1" +} +` + +const testAccESDomainConfig_complex = ` +resource "aws_elasticsearch_domain" "example" { + domain_name = "tf-test-2" + + advanced_options { + "indices.fielddata.cache.size" = 80 + } + + ebs_options { + ebs_enabled = false + } + + cluster_config { + instance_count = 2 + zone_awareness_enabled = true + } + + snapshot_options { + automated_snapshot_start_hour = 23 + } +} +` From e65ef8f13ff2289de85ee34b1def591f416817ef Mon Sep 17 00:00:00 2001 From: Radek Simko Date: Thu, 1 Oct 2015 15:13:32 -0700 Subject: [PATCH 3/3] provider/aws: Add docs for aws_elasticsearch_domain --- .../aws/r/elasticsearch_domain.html.markdown | 83 +++++++++++++++++++ website/source/layouts/aws.erb | 12 +++ 2 files changed, 95 insertions(+) create mode 100644 website/source/docs/providers/aws/r/elasticsearch_domain.html.markdown diff --git a/website/source/docs/providers/aws/r/elasticsearch_domain.html.markdown b/website/source/docs/providers/aws/r/elasticsearch_domain.html.markdown new file mode 100644 index 000000000..9dbacffcd --- /dev/null +++ b/website/source/docs/providers/aws/r/elasticsearch_domain.html.markdown @@ -0,0 +1,83 @@ +--- +layout: "aws" +page_title: "AWS: aws_elasticsearch_domain" +sidebar_current: "docs-aws-elasticsearch-domain" +description: |- + Provides an ElasticSearch Domain. +--- + +# aws\_elasticsearch\_domain + + +## Example Usage + +``` +resource "aws_elasticsearch_domain" "es" { + domain_name = "tf-test" + advanced_options { + "rest.action.multi.allow_explicit_index" = true + } + + access_policies = < + > + ElasticSearch Resources + + + + > IAM Resources