provider/aws: Fix DynamoDB issues about GSIs indexes (#13256)

* provider/aws: Fixed DynamoDB GSI update when using multiple indexes

* provider/aws: Fixed DynamoDB GSI set hash function

* Added DynamoDB table state migration
This commit is contained in:
Gauthier Wallet 2017-04-07 17:13:00 +02:00 committed by Clint
parent 19eecd4a71
commit c44afc4179
3 changed files with 434 additions and 159 deletions

View File

@ -39,6 +39,9 @@ func resourceAwsDynamoDbTable() *schema.Resource {
State: schema.ImportStatePassthrough, State: schema.ImportStatePassthrough,
}, },
SchemaVersion: 1,
MigrateState: resourceAwsDynamoDbTableMigrateState,
Schema: map[string]*schema.Schema{ Schema: map[string]*schema.Schema{
"arn": { "arn": {
Type: schema.TypeString, Type: schema.TypeString,
@ -157,15 +160,6 @@ func resourceAwsDynamoDbTable() *schema.Resource {
}, },
}, },
}, },
// GSI names are the uniqueness constraint
Set: func(v interface{}) int {
var buf bytes.Buffer
m := v.(map[string]interface{})
buf.WriteString(fmt.Sprintf("%s-", m["name"].(string)))
buf.WriteString(fmt.Sprintf("%d-", m["write_capacity"].(int)))
buf.WriteString(fmt.Sprintf("%d-", m["read_capacity"].(int)))
return hashcode.String(buf.String())
},
}, },
"stream_enabled": { "stream_enabled": {
Type: schema.TypeBool, Type: schema.TypeBool,
@ -533,9 +527,8 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er
table := tableDescription.Table table := tableDescription.Table
updates := []*dynamodb.GlobalSecondaryIndexUpdate{}
for _, updatedgsidata := range gsiSet.List() { for _, updatedgsidata := range gsiSet.List() {
updates := []*dynamodb.GlobalSecondaryIndexUpdate{}
gsidata := updatedgsidata.(map[string]interface{}) gsidata := updatedgsidata.(map[string]interface{})
gsiName := gsidata["name"].(string) gsiName := gsidata["name"].(string)
gsiWriteCapacity := gsidata["write_capacity"].(int) gsiWriteCapacity := gsidata["write_capacity"].(int)
@ -584,6 +577,10 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er
log.Printf("[DEBUG] Error updating table: %s", err) log.Printf("[DEBUG] Error updating table: %s", err)
return err return err
} }
if err := waitForGSIToBeActive(d.Id(), gsiName, meta); err != nil {
return errwrap.Wrapf("Error waiting for Dynamo DB GSI to be active: {{err}}", err)
}
} }
} }
} }

View File

@ -0,0 +1,70 @@
package aws
import (
"fmt"
"log"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
"strings"
)
func resourceAwsDynamoDbTableMigrateState(
v int, is *terraform.InstanceState, meta interface{}) (*terraform.InstanceState, error) {
switch v {
case 0:
log.Println("[INFO] Found AWS DynamoDB Table State v0; migrating to v1")
return migrateDynamoDBStateV0toV1(is)
default:
return is, fmt.Errorf("Unexpected schema version: %d", v)
}
}
func migrateDynamoDBStateV0toV1(is *terraform.InstanceState) (*terraform.InstanceState, error) {
if is.Empty() {
log.Println("[DEBUG] Empty InstanceState; nothing to migrate.")
return is, nil
}
log.Printf("[DEBUG] DynamoDB Table Attributes before Migration: %#v", is.Attributes)
prefix := "global_secondary_index"
entity := resourceAwsDynamoDbTable()
// Read old keys
reader := &schema.MapFieldReader{
Schema: entity.Schema,
Map: schema.BasicMapReader(is.Attributes),
}
result, err := reader.ReadField([]string{prefix})
if err != nil {
return nil, err
}
oldKeys, ok := result.Value.(*schema.Set)
if !ok {
return nil, fmt.Errorf("Got unexpected value from state: %#v", result.Value)
}
// Delete old keys
for k := range is.Attributes {
if strings.HasPrefix(k, fmt.Sprintf("%s.", prefix)) {
delete(is.Attributes, k)
}
}
// Write new keys
writer := schema.MapFieldWriter{
Schema: entity.Schema,
}
if err := writer.WriteField([]string{prefix}, oldKeys); err != nil {
return is, err
}
for k, v := range writer.Map() {
is.Attributes[k] = v
}
log.Printf("[DEBUG] DynamoDB Table Attributes after State Migration: %#v", is.Attributes)
return is, nil
}

View File

@ -14,6 +14,8 @@ import (
) )
func TestAccAWSDynamoDbTable_basic(t *testing.T) { func TestAccAWSDynamoDbTable_basic(t *testing.T) {
var conf dynamodb.DescribeTableOutput
resource.Test(t, resource.TestCase{ resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) }, PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders, Providers: testAccProviders,
@ -22,7 +24,8 @@ func TestAccAWSDynamoDbTable_basic(t *testing.T) {
{ {
Config: testAccAWSDynamoDbConfigInitialState(), Config: testAccAWSDynamoDbConfigInitialState(),
Check: resource.ComposeTestCheckFunc( Check: resource.ComposeTestCheckFunc(
testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table"), testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table", &conf),
testAccCheckInitialAWSDynamoDbTableConf("aws_dynamodb_table.basic-dynamodb-table"),
), ),
}, },
{ {
@ -36,6 +39,8 @@ func TestAccAWSDynamoDbTable_basic(t *testing.T) {
} }
func TestAccAWSDynamoDbTable_streamSpecification(t *testing.T) { func TestAccAWSDynamoDbTable_streamSpecification(t *testing.T) {
var conf dynamodb.DescribeTableOutput
resource.Test(t, resource.TestCase{ resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) }, PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders, Providers: testAccProviders,
@ -44,7 +49,8 @@ func TestAccAWSDynamoDbTable_streamSpecification(t *testing.T) {
{ {
Config: testAccAWSDynamoDbConfigStreamSpecification(), Config: testAccAWSDynamoDbConfigStreamSpecification(),
Check: resource.ComposeTestCheckFunc( Check: resource.ComposeTestCheckFunc(
testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table"), testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table", &conf),
testAccCheckInitialAWSDynamoDbTableConf("aws_dynamodb_table.basic-dynamodb-table"),
resource.TestCheckResourceAttr( resource.TestCheckResourceAttr(
"aws_dynamodb_table.basic-dynamodb-table", "stream_enabled", "true"), "aws_dynamodb_table.basic-dynamodb-table", "stream_enabled", "true"),
resource.TestCheckResourceAttr( resource.TestCheckResourceAttr(
@ -56,6 +62,8 @@ func TestAccAWSDynamoDbTable_streamSpecification(t *testing.T) {
} }
func TestAccAWSDynamoDbTable_tags(t *testing.T) { func TestAccAWSDynamoDbTable_tags(t *testing.T) {
var conf dynamodb.DescribeTableOutput
resource.Test(t, resource.TestCase{ resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) }, PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders, Providers: testAccProviders,
@ -64,7 +72,8 @@ func TestAccAWSDynamoDbTable_tags(t *testing.T) {
{ {
Config: testAccAWSDynamoDbConfigTags(), Config: testAccAWSDynamoDbConfigTags(),
Check: resource.ComposeTestCheckFunc( Check: resource.ComposeTestCheckFunc(
testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table"), testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table", &conf),
testAccCheckInitialAWSDynamoDbTableConf("aws_dynamodb_table.basic-dynamodb-table"),
resource.TestCheckResourceAttr( resource.TestCheckResourceAttr(
"aws_dynamodb_table.basic-dynamodb-table", "tags.%", "3"), "aws_dynamodb_table.basic-dynamodb-table", "tags.%", "3"),
), ),
@ -73,6 +82,32 @@ func TestAccAWSDynamoDbTable_tags(t *testing.T) {
}) })
} }
// https://github.com/hashicorp/terraform/issues/13243
func TestAccAWSDynamoDbTable_gsiUpdate(t *testing.T) {
var conf dynamodb.DescribeTableOutput
name := acctest.RandString(10)
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSDynamoDbTableDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSDynamoDbConfigGsiUpdate(name),
Check: resource.ComposeTestCheckFunc(
testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.test", &conf),
),
},
{
Config: testAccAWSDynamoDbConfigGsiUpdated(name),
Check: resource.ComposeTestCheckFunc(
testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.test", &conf),
),
},
},
})
}
func TestResourceAWSDynamoDbTableStreamViewType_validation(t *testing.T) { func TestResourceAWSDynamoDbTableStreamViewType_validation(t *testing.T) {
cases := []struct { cases := []struct {
Value string Value string
@ -143,7 +178,37 @@ func testAccCheckAWSDynamoDbTableDestroy(s *terraform.State) error {
return nil return nil
} }
func testAccCheckInitialAWSDynamoDbTableExists(n string) resource.TestCheckFunc { func testAccCheckInitialAWSDynamoDbTableExists(n string, table *dynamodb.DescribeTableOutput) resource.TestCheckFunc {
return func(s *terraform.State) error {
log.Printf("[DEBUG] Trying to create initial table state!")
rs, ok := s.RootModule().Resources[n]
if !ok {
return fmt.Errorf("Not found: %s", n)
}
if rs.Primary.ID == "" {
return fmt.Errorf("No DynamoDB table name specified!")
}
conn := testAccProvider.Meta().(*AWSClient).dynamodbconn
params := &dynamodb.DescribeTableInput{
TableName: aws.String(rs.Primary.ID),
}
resp, err := conn.DescribeTable(params)
if err != nil {
return fmt.Errorf("[ERROR] Problem describing table '%s': %s", rs.Primary.ID, err)
}
*table = *resp
return nil
}
}
func testAccCheckInitialAWSDynamoDbTableConf(n string) resource.TestCheckFunc {
return func(s *terraform.State) error { return func(s *terraform.State) error {
log.Printf("[DEBUG] Trying to create initial table state!") log.Printf("[DEBUG] Trying to create initial table state!")
rs, ok := s.RootModule().Resources[n] rs, ok := s.RootModule().Resources[n]
@ -301,123 +366,141 @@ func dynamoDbAttributesToMap(attributes *[]*dynamodb.AttributeDefinition) map[st
func testAccAWSDynamoDbConfigInitialState() string { func testAccAWSDynamoDbConfigInitialState() string {
return fmt.Sprintf(` return fmt.Sprintf(`
resource "aws_dynamodb_table" "basic-dynamodb-table" { resource "aws_dynamodb_table" "basic-dynamodb-table" {
name = "TerraformTestTable-%d" name = "TerraformTestTable-%d"
read_capacity = 10 read_capacity = 10
write_capacity = 20 write_capacity = 20
hash_key = "TestTableHashKey" hash_key = "TestTableHashKey"
range_key = "TestTableRangeKey" range_key = "TestTableRangeKey"
attribute {
name = "TestTableHashKey" attribute {
type = "S" name = "TestTableHashKey"
} type = "S"
attribute { }
name = "TestTableRangeKey"
type = "S" attribute {
} name = "TestTableRangeKey"
attribute { type = "S"
name = "TestLSIRangeKey" }
type = "N"
} attribute {
attribute { name = "TestLSIRangeKey"
name = "TestGSIRangeKey" type = "N"
type = "S" }
}
local_secondary_index { attribute {
name = "TestTableLSI" name = "TestGSIRangeKey"
range_key = "TestLSIRangeKey" type = "S"
projection_type = "ALL" }
}
global_secondary_index { local_secondary_index {
name = "InitialTestTableGSI" name = "TestTableLSI"
hash_key = "TestTableHashKey" range_key = "TestLSIRangeKey"
range_key = "TestGSIRangeKey" projection_type = "ALL"
write_capacity = 10 }
read_capacity = 10
projection_type = "KEYS_ONLY" global_secondary_index {
} name = "InitialTestTableGSI"
hash_key = "TestTableHashKey"
range_key = "TestGSIRangeKey"
write_capacity = 10
read_capacity = 10
projection_type = "KEYS_ONLY"
}
} }
`, acctest.RandInt()) `, acctest.RandInt())
} }
const testAccAWSDynamoDbConfigAddSecondaryGSI = ` const testAccAWSDynamoDbConfigAddSecondaryGSI = `
resource "aws_dynamodb_table" "basic-dynamodb-table" { resource "aws_dynamodb_table" "basic-dynamodb-table" {
name = "TerraformTestTable" name = "TerraformTestTable"
read_capacity = 20 read_capacity = 20
write_capacity = 20 write_capacity = 20
hash_key = "TestTableHashKey" hash_key = "TestTableHashKey"
range_key = "TestTableRangeKey" range_key = "TestTableRangeKey"
attribute {
name = "TestTableHashKey" attribute {
type = "S" name = "TestTableHashKey"
} type = "S"
attribute { }
name = "TestTableRangeKey"
type = "S" attribute {
} name = "TestTableRangeKey"
attribute { type = "S"
name = "TestLSIRangeKey" }
type = "N"
} attribute {
attribute { name = "TestLSIRangeKey"
name = "ReplacementGSIRangeKey" type = "N"
type = "N" }
}
local_secondary_index { attribute {
name = "TestTableLSI" name = "ReplacementGSIRangeKey"
range_key = "TestLSIRangeKey" type = "N"
projection_type = "ALL" }
}
global_secondary_index { local_secondary_index {
name = "ReplacementTestTableGSI" name = "TestTableLSI"
hash_key = "TestTableHashKey" range_key = "TestLSIRangeKey"
range_key = "ReplacementGSIRangeKey" projection_type = "ALL"
write_capacity = 5 }
read_capacity = 5
projection_type = "INCLUDE" global_secondary_index {
non_key_attributes = ["TestNonKeyAttribute"] name = "ReplacementTestTableGSI"
} hash_key = "TestTableHashKey"
range_key = "ReplacementGSIRangeKey"
write_capacity = 5
read_capacity = 5
projection_type = "INCLUDE"
non_key_attributes = ["TestNonKeyAttribute"]
}
} }
` `
func testAccAWSDynamoDbConfigStreamSpecification() string { func testAccAWSDynamoDbConfigStreamSpecification() string {
return fmt.Sprintf(` return fmt.Sprintf(`
resource "aws_dynamodb_table" "basic-dynamodb-table" { resource "aws_dynamodb_table" "basic-dynamodb-table" {
name = "TerraformTestStreamTable-%d" name = "TerraformTestStreamTable-%d"
read_capacity = 10 read_capacity = 10
write_capacity = 20 write_capacity = 20
hash_key = "TestTableHashKey" hash_key = "TestTableHashKey"
range_key = "TestTableRangeKey" range_key = "TestTableRangeKey"
attribute {
name = "TestTableHashKey" attribute {
type = "S" name = "TestTableHashKey"
} type = "S"
attribute { }
name = "TestTableRangeKey"
type = "S" attribute {
} name = "TestTableRangeKey"
attribute { type = "S"
name = "TestLSIRangeKey" }
type = "N"
} attribute {
attribute { name = "TestLSIRangeKey"
name = "TestGSIRangeKey" type = "N"
type = "S" }
}
local_secondary_index { attribute {
name = "TestTableLSI" name = "TestGSIRangeKey"
range_key = "TestLSIRangeKey" type = "S"
projection_type = "ALL" }
}
global_secondary_index { local_secondary_index {
name = "InitialTestTableGSI" name = "TestTableLSI"
hash_key = "TestTableHashKey" range_key = "TestLSIRangeKey"
range_key = "TestGSIRangeKey" projection_type = "ALL"
write_capacity = 10 }
read_capacity = 10
projection_type = "KEYS_ONLY" global_secondary_index {
} name = "InitialTestTableGSI"
stream_enabled = true hash_key = "TestTableHashKey"
stream_view_type = "KEYS_ONLY" range_key = "TestGSIRangeKey"
write_capacity = 10
read_capacity = 10
projection_type = "KEYS_ONLY"
}
stream_enabled = true
stream_view_type = "KEYS_ONLY"
} }
`, acctest.RandInt()) `, acctest.RandInt())
} }
@ -425,45 +508,170 @@ resource "aws_dynamodb_table" "basic-dynamodb-table" {
func testAccAWSDynamoDbConfigTags() string { func testAccAWSDynamoDbConfigTags() string {
return fmt.Sprintf(` return fmt.Sprintf(`
resource "aws_dynamodb_table" "basic-dynamodb-table" { resource "aws_dynamodb_table" "basic-dynamodb-table" {
name = "TerraformTestTable-%d" name = "TerraformTestTable-%d"
read_capacity = 10 read_capacity = 10
write_capacity = 20 write_capacity = 20
hash_key = "TestTableHashKey" hash_key = "TestTableHashKey"
range_key = "TestTableRangeKey" range_key = "TestTableRangeKey"
attribute {
name = "TestTableHashKey" attribute {
type = "S" name = "TestTableHashKey"
} type = "S"
attribute { }
name = "TestTableRangeKey"
type = "S" attribute {
} name = "TestTableRangeKey"
attribute { type = "S"
name = "TestLSIRangeKey" }
type = "N"
} attribute {
attribute { name = "TestLSIRangeKey"
name = "TestGSIRangeKey" type = "N"
type = "S" }
}
local_secondary_index { attribute {
name = "TestTableLSI" name = "TestGSIRangeKey"
range_key = "TestLSIRangeKey" type = "S"
projection_type = "ALL" }
}
global_secondary_index { local_secondary_index {
name = "InitialTestTableGSI" name = "TestTableLSI"
hash_key = "TestTableHashKey" range_key = "TestLSIRangeKey"
range_key = "TestGSIRangeKey" projection_type = "ALL"
write_capacity = 10 }
read_capacity = 10
projection_type = "KEYS_ONLY" global_secondary_index {
} name = "InitialTestTableGSI"
tags { hash_key = "TestTableHashKey"
Name = "terraform-test-table-%d" range_key = "TestGSIRangeKey"
AccTest = "yes" write_capacity = 10
Testing = "absolutely" read_capacity = 10
} projection_type = "KEYS_ONLY"
}
tags {
Name = "terraform-test-table-%d"
AccTest = "yes"
Testing = "absolutely"
}
} }
`, acctest.RandInt(), acctest.RandInt()) `, acctest.RandInt(), acctest.RandInt())
} }
func testAccAWSDynamoDbConfigGsiUpdate(name string) string {
return fmt.Sprintf(`
variable "capacity" {
default = 10
}
resource "aws_dynamodb_table" "test" {
name = "tf-acc-test-%s"
read_capacity = "${var.capacity}"
write_capacity = "${var.capacity}"
hash_key = "id"
attribute {
name = "id"
type = "S"
}
attribute {
name = "att1"
type = "S"
}
attribute {
name = "att2"
type = "S"
}
attribute {
name = "att3"
type = "S"
}
global_secondary_index {
name = "att1-index"
hash_key = "att1"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
global_secondary_index {
name = "att2-index"
hash_key = "att2"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
global_secondary_index {
name = "att3-index"
hash_key = "att3"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
}
`, name)
}
func testAccAWSDynamoDbConfigGsiUpdated(name string) string {
return fmt.Sprintf(`
variable "capacity" {
default = 20
}
resource "aws_dynamodb_table" "test" {
name = "tf-acc-test-%s"
read_capacity = "${var.capacity}"
write_capacity = "${var.capacity}"
hash_key = "id"
attribute {
name = "id"
type = "S"
}
attribute {
name = "att1"
type = "S"
}
attribute {
name = "att2"
type = "S"
}
attribute {
name = "att3"
type = "S"
}
global_secondary_index {
name = "att1-index"
hash_key = "att1"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
global_secondary_index {
name = "att2-index"
hash_key = "att2"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
global_secondary_index {
name = "att3-index"
hash_key = "att3"
write_capacity = "${var.capacity}"
read_capacity = "${var.capacity}"
projection_type = "ALL"
}
}
`, name)
}