From d706130a51e95a6518b49944e3739b27bbd1410b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 2 Apr 2016 20:37:11 -0700 Subject: [PATCH] consul_key_prefix resource This new resource is an alternative to consul_keys that manages all keys under a given prefix, rather than arbitrary single keys across the entire store. The key advantage of this resource over consul_keys is that it is able to detect and delete keys that are added outside of Terraform, whereas consul_keys is only able to detect changes to keys it is explicitly managing. --- builtin/providers/consul/key_client.go | 30 +++ .../consul/resource_consul_key_prefix.go | 221 ++++++++++++++++++ .../consul/resource_consul_key_prefix_test.go | 150 ++++++++++++ builtin/providers/consul/resource_provider.go | 3 +- .../consul/r/key_prefix.html.markdown | 79 +++++++ .../providers/consul/r/keys.html.markdown | 7 + 6 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 builtin/providers/consul/resource_consul_key_prefix.go create mode 100644 builtin/providers/consul/resource_consul_key_prefix_test.go create mode 100644 website/source/docs/providers/consul/r/key_prefix.html.markdown diff --git a/builtin/providers/consul/key_client.go b/builtin/providers/consul/key_client.go index 5f41c8df7..0b909dd2e 100644 --- a/builtin/providers/consul/key_client.go +++ b/builtin/providers/consul/key_client.go @@ -42,6 +42,25 @@ func (c *keyClient) Get(path string) (string, error) { return value, nil } +func (c *keyClient) GetUnderPrefix(pathPrefix string) (map[string]string, error) { + log.Printf( + "[DEBUG] Listing keys under '%s' in %s", + pathPrefix, c.qOpts.Datacenter, + ) + pairs, _, err := c.client.List(pathPrefix, c.qOpts) + if err != nil { + return nil, fmt.Errorf( + "Failed to list Consul keys under prefix '%s': %s", pathPrefix, err, + ) + } + value := map[string]string{} + for _, pair := range pairs { + subKey := pair.Key[len(pathPrefix):] + value[subKey] = string(pair.Value) + } + return value, nil +} + func (c *keyClient) Put(path, value string) error { log.Printf( "[DEBUG] Setting key '%s' to '%v' in %s", @@ -64,3 +83,14 @@ func (c *keyClient) Delete(path string) error { } return nil } + +func (c *keyClient) DeleteUnderPrefix(pathPrefix string) error { + log.Printf( + "[DEBUG] Deleting all keys under prefix '%s' in %s", + pathPrefix, c.wOpts.Datacenter, + ) + if _, err := c.client.DeleteTree(pathPrefix, c.wOpts); err != nil { + return fmt.Errorf("Failed to delete Consul keys under '%s': %s", pathPrefix, err) + } + return nil +} diff --git a/builtin/providers/consul/resource_consul_key_prefix.go b/builtin/providers/consul/resource_consul_key_prefix.go new file mode 100644 index 000000000..f16460309 --- /dev/null +++ b/builtin/providers/consul/resource_consul_key_prefix.go @@ -0,0 +1,221 @@ +package consul + +import ( + "fmt" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceConsulKeyPrefix() *schema.Resource { + return &schema.Resource{ + Create: resourceConsulKeyPrefixCreate, + Update: resourceConsulKeyPrefixUpdate, + Read: resourceConsulKeyPrefixRead, + Delete: resourceConsulKeyPrefixDelete, + + Schema: map[string]*schema.Schema{ + "datacenter": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + ForceNew: true, + }, + + "token": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + + "path_prefix": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + + "subkeys": &schema.Schema{ + Type: schema.TypeMap, + Required: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + } +} + +func resourceConsulKeyPrefixCreate(d *schema.ResourceData, meta interface{}) error { + client := meta.(*consulapi.Client) + kv := client.KV() + token := d.Get("token").(string) + dc, err := getDC(d, client) + if err != nil { + return err + } + + keyClient := newKeyClient(kv, dc, token) + + pathPrefix := d.Get("path_prefix").(string) + subKeys := map[string]string{} + for k, vI := range d.Get("subkeys").(map[string]interface{}) { + subKeys[k] = vI.(string) + } + + // To reduce the impact of mistakes, we will only "create" a prefix that + // is currently empty. This way we are less likely to accidentally + // conflict with other mechanisms managing the same prefix. + currentSubKeys, err := keyClient.GetUnderPrefix(pathPrefix) + if err != nil { + return err + } + if len(currentSubKeys) > 0 { + return fmt.Errorf( + "%d keys already exist under %s; delete them before managing this prefix with Terraform", + len(currentSubKeys), pathPrefix, + ) + } + + // Ideally we'd use d.Partial(true) here so we can correctly record + // a partial write, but that mechanism doesn't work for individual map + // members, so we record that the resource was created before we + // do anything and that way we can recover from errors by doing an + // Update on subsequent runs, rather than re-attempting Create with + // some keys possibly already present. + d.SetId(pathPrefix) + + // Store the datacenter on this resource, which can be helpful for reference + // in case it was read from the provider + d.Set("datacenter", dc) + + // Now we can just write in all the initial values, since we can expect + // that nothing should need deleting yet, as long as there isn't some + // other program racing us to write values... which we'll catch on a + // subsequent Read. + for k, v := range subKeys { + fullPath := pathPrefix + k + err := keyClient.Put(fullPath, v) + if err != nil { + return fmt.Errorf("error while writing %s: %s", fullPath, err) + } + } + + return nil +} + +func resourceConsulKeyPrefixUpdate(d *schema.ResourceData, meta interface{}) error { + client := meta.(*consulapi.Client) + kv := client.KV() + token := d.Get("token").(string) + dc, err := getDC(d, client) + if err != nil { + return err + } + + keyClient := newKeyClient(kv, dc, token) + + pathPrefix := d.Id() + + if d.HasChange("subkeys") { + o, n := d.GetChange("subkeys") + if o == nil { + o = map[string]interface{}{} + } + if n == nil { + n = map[string]interface{}{} + } + + om := o.(map[string]interface{}) + nm := n.(map[string]interface{}) + + // First we'll write all of the stuff in the "new map" nm, + // and then we'll delete any keys that appear in the "old map" om + // and do not also appear in nm. This ordering means that if a subkey + // name is changed we will briefly have both the old and new names in + // Consul, as opposed to briefly having neither. + + // Again, we'd ideally use d.Partial(true) here but it doesn't work + // for maps and so we'll just rely on a subsequent Read to tidy up + // after a partial write. + + // Write new and changed keys + for k, vI := range nm { + v := vI.(string) + fullPath := pathPrefix + k + err := keyClient.Put(fullPath, v) + if err != nil { + return fmt.Errorf("error while writing %s: %s", fullPath, err) + } + } + + // Remove deleted keys + for k, _ := range om { + if _, exists := nm[k]; exists { + continue + } + fullPath := pathPrefix + k + err := keyClient.Delete(fullPath) + if err != nil { + return fmt.Errorf("error while deleting %s: %s", fullPath, err) + } + } + + } + + // Store the datacenter on this resource, which can be helpful for reference + // in case it was read from the provider + d.Set("datacenter", dc) + + return nil +} + +func resourceConsulKeyPrefixRead(d *schema.ResourceData, meta interface{}) error { + client := meta.(*consulapi.Client) + kv := client.KV() + token := d.Get("token").(string) + dc, err := getDC(d, client) + if err != nil { + return err + } + + keyClient := newKeyClient(kv, dc, token) + + pathPrefix := d.Id() + + subKeys, err := keyClient.GetUnderPrefix(pathPrefix) + if err != nil { + return err + } + + d.Set("subkeys", subKeys) + + // Store the datacenter on this resource, which can be helpful for reference + // in case it was read from the provider + d.Set("datacenter", dc) + + return nil +} + +func resourceConsulKeyPrefixDelete(d *schema.ResourceData, meta interface{}) error { + client := meta.(*consulapi.Client) + kv := client.KV() + token := d.Get("token").(string) + dc, err := getDC(d, client) + if err != nil { + return err + } + + keyClient := newKeyClient(kv, dc, token) + + pathPrefix := d.Id() + + // Delete everything under our prefix, since the entire set of keys under + // the given prefix is considered to be managed exclusively by Terraform. + err = keyClient.DeleteUnderPrefix(pathPrefix) + if err != nil { + return err + } + + d.SetId("") + + return nil +} diff --git a/builtin/providers/consul/resource_consul_key_prefix_test.go b/builtin/providers/consul/resource_consul_key_prefix_test.go new file mode 100644 index 000000000..e33fb13da --- /dev/null +++ b/builtin/providers/consul/resource_consul_key_prefix_test.go @@ -0,0 +1,150 @@ +package consul + +import ( + "fmt" + "testing" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccConsulKeyPrefix_basic(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: resource.ComposeTestCheckFunc( + testAccCheckConsulKeyPrefixKeyAbsent("species"), + testAccCheckConsulKeyPrefixKeyAbsent("meat"), + testAccCheckConsulKeyPrefixKeyAbsent("cheese"), + testAccCheckConsulKeyPrefixKeyAbsent("bread"), + ), + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccConsulKeyPrefixConfig, + Check: resource.ComposeTestCheckFunc( + testAccCheckConsulKeyPrefixKeyValue("cheese", "chevre"), + testAccCheckConsulKeyPrefixKeyValue("bread", "baguette"), + testAccCheckConsulKeyPrefixKeyAbsent("species"), + testAccCheckConsulKeyPrefixKeyAbsent("meat"), + ), + }, + resource.TestStep{ + Config: testAccConsulKeyPrefixConfig, + ExpectNonEmptyPlan: true, + Check: resource.ComposeTestCheckFunc( + // This will add a rogue key that Terraform isn't + // expecting, causing a non-empty plan that wants + // to remove it. + testAccAddConsulKeyPrefixRogue("species", "gorilla"), + ), + }, + resource.TestStep{ + Config: testAccConsulKeyPrefixConfig_Update, + Check: resource.ComposeTestCheckFunc( + testAccCheckConsulKeyPrefixKeyValue("meat", "ham"), + testAccCheckConsulKeyPrefixKeyValue("bread", "batard"), + testAccCheckConsulKeyPrefixKeyAbsent("cheese"), + testAccCheckConsulKeyPrefixKeyAbsent("species"), + ), + }, + resource.TestStep{ + Config: testAccConsulKeyPrefixConfig_Update, + ExpectNonEmptyPlan: true, + Check: resource.ComposeTestCheckFunc( + testAccAddConsulKeyPrefixRogue("species", "gorilla"), + ), + }, + }, + }) +} + +func testAccCheckConsulKeyPrefixDestroy(s *terraform.State) error { + kv := testAccProvider.Meta().(*consulapi.Client).KV() + opts := &consulapi.QueryOptions{Datacenter: "dc1"} + pair, _, err := kv.Get("test/set", opts) + if err != nil { + return err + } + if pair != nil { + return fmt.Errorf("Key still exists: %#v", pair) + } + return nil +} + +func testAccCheckConsulKeyPrefixKeyAbsent(name string) resource.TestCheckFunc { + fullName := "prefix_test/" + name + return func(s *terraform.State) error { + kv := testAccProvider.Meta().(*consulapi.Client).KV() + opts := &consulapi.QueryOptions{Datacenter: "dc1"} + pair, _, err := kv.Get(fullName, opts) + if err != nil { + return err + } + if pair != nil { + return fmt.Errorf("key '%s' exists, but shouldn't", fullName) + } + return nil + } +} + +// This one is actually not a check, but rather a mutation step. It writes +// a value directly into Consul, bypassing our Terraform resource. +func testAccAddConsulKeyPrefixRogue(name, value string) resource.TestCheckFunc { + fullName := "prefix_test/" + name + return func(s *terraform.State) error { + kv := testAccProvider.Meta().(*consulapi.Client).KV() + opts := &consulapi.WriteOptions{Datacenter: "dc1"} + pair := &consulapi.KVPair{ + Key: fullName, + Value: []byte(value), + } + _, err := kv.Put(pair, opts) + return err + } +} + +func testAccCheckConsulKeyPrefixKeyValue(name, value string) resource.TestCheckFunc { + fullName := "prefix_test/" + name + return func(s *terraform.State) error { + kv := testAccProvider.Meta().(*consulapi.Client).KV() + opts := &consulapi.QueryOptions{Datacenter: "dc1"} + pair, _, err := kv.Get(fullName, opts) + if err != nil { + return err + } + if pair == nil { + return fmt.Errorf("key %v doesn't exist, but should", fullName) + } + if string(pair.Value) != value { + return fmt.Errorf("key %v has value %v; want %v", fullName, pair.Value, value) + } + return nil + } +} + +const testAccConsulKeyPrefixConfig = ` +resource "consul_key_prefix" "app" { + datacenter = "dc1" + + path_prefix = "prefix_test/" + + subkeys = { + cheese = "chevre" + bread = "baguette" + } +} +` + +const testAccConsulKeyPrefixConfig_Update = ` +resource "consul_key_prefix" "app" { + datacenter = "dc1" + + path_prefix = "prefix_test/" + + subkeys = { + bread = "batard" + meat = "ham" + } +} +` diff --git a/builtin/providers/consul/resource_provider.go b/builtin/providers/consul/resource_provider.go index ec5cd43a5..d2f8d269a 100644 --- a/builtin/providers/consul/resource_provider.go +++ b/builtin/providers/consul/resource_provider.go @@ -29,7 +29,8 @@ func Provider() terraform.ResourceProvider { }, ResourcesMap: map[string]*schema.Resource{ - "consul_keys": resourceConsulKeys(), + "consul_keys": resourceConsulKeys(), + "consul_key_prefix": resourceConsulKeyPrefix(), }, ConfigureFunc: providerConfigure, diff --git a/website/source/docs/providers/consul/r/key_prefix.html.markdown b/website/source/docs/providers/consul/r/key_prefix.html.markdown new file mode 100644 index 000000000..5a8d0fded --- /dev/null +++ b/website/source/docs/providers/consul/r/key_prefix.html.markdown @@ -0,0 +1,79 @@ +--- +layout: "consul" +page_title: "Consul: consul_key_prefix" +sidebar_current: "docs-consul-resource-key-prefix" +description: |- + Allows Terraform to manage a namespace of Consul keys that share a + common name prefix. +--- + +# consul\_key\_prefix + +Allows Terraform to manage a "namespace" of Consul keys that share a common +name prefix. + +Like `consul_keys`, this resource can write values into the Consul key/value +store, but *unlike* `consul_keys` this resource can detect and remove extra +keys that have been added some other way, thus ensuring that rogue data +added outside of Terraform will be removed on the next run. + +This resource is thus useful in the case where Terraform is exclusively +managing a set of related keys. + +To avoid accidentally clobbering matching data that existed in Consul before +a `consul_key_prefix` resource was created, creation of a key prefix instance +will fail if any matching keys are already present in the key/value store. +If any conflicting data is present, you must first delete it manually. + +~> **Warning** After this resource is instantiated, Terraform takes control +over *all* keys with the given path prefix, and will remove any matching keys +that are not present in the configuration. It will also delete *all* keys under +the given prefix when a `consul_key_prefix` resource is destroyed, even if +those keys were created outside of Terraform. + +## Example Usage + +``` +resource "consul_key_prefix" "myapp_config" { + datacenter = "nyc1" + token = "abcd" + + # Prefix to add to prepend to all of the subkey names below. + path_prefix = "myapp/config/" + + subkeys = { + "elb_cname" = "${aws_elb.app.dns_name}" + "s3_bucket_name" = "${aws_s3_bucket.app.bucket}" + "database/hostname" = "${aws_db_instance.app.address}" + "database/port" = "${aws_db_instance.app.port}" + "database/username" = "${aws_db_instance.app.username}" + "database/password" = "${aws_db_instance.app.password}" + "database/name" = "${aws_db_instance.app.name}" + } +} +``` + +## Argument Reference + +The following arguments are supported: + +* `datacenter` - (Optional) The datacenter to use. This overrides the + datacenter in the provider setup and the agent's default datacenter. + +* `token` - (Optional) The ACL token to use. This overrides the + token that the agent provides by default. + +* `path_prefix` - (Required) Specifies the common prefix shared by all keys + that will be managed by this resource instance. In most cases this will + end with a slash, to manage a "folder" of keys. + +* `subkeys` - (Required) A mapping from subkey name (which will be appended + to the give `path_prefix`) to the value that should be stored at that key. + Use slashes as shown in the above example to create "sub-folders" under + the given path prefix. + +## Attributes Reference + +The following attributes are exported: + +* `datacenter` - The datacenter the keys are being read/written to. diff --git a/website/source/docs/providers/consul/r/keys.html.markdown b/website/source/docs/providers/consul/r/keys.html.markdown index c1c806961..3fcd66ac5 100644 --- a/website/source/docs/providers/consul/r/keys.html.markdown +++ b/website/source/docs/providers/consul/r/keys.html.markdown @@ -13,6 +13,13 @@ to both read keys from Consul, but also to set the value of keys in Consul. This is a powerful way dynamically set values in templates, and to expose infrastructure details to clients. +This resource manages individual keys, and thus it can create, update and +delete the keys explicitly given. Howver, It is not able to detect and remove +additional keys that have been added by non-Terraform means. To manage +*all* keys sharing a common prefix, and thus have Terraform remove errant keys +not present in the configuration, consider using the `consul_key_prefix` +resource instead. + ## Example Usage ```