Merge pull request #5988 from apparentlymart/consul-key-subtree

provider/consul: consul_key_prefix resource

builtin/providers/consul/key_client.go

@@ -42,6 +42,25 @@ func (c *keyClient) Get(path string) (string, error) {
 	return value, nil
 }
 
+func (c *keyClient) GetUnderPrefix(pathPrefix string) (map[string]string, error) {
+	log.Printf(
+		"[DEBUG] Listing keys under '%s' in %s",
+		pathPrefix, c.qOpts.Datacenter,
+	)
+	pairs, _, err := c.client.List(pathPrefix, c.qOpts)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"Failed to list Consul keys under prefix '%s': %s", pathPrefix, err,
+		)
+	}
+	value := map[string]string{}
+	for _, pair := range pairs {
+		subKey := pair.Key[len(pathPrefix):]
+		value[subKey] = string(pair.Value)
+	}
+	return value, nil
+}
+
 func (c *keyClient) Put(path, value string) error {
 	log.Printf(
 		"[DEBUG] Setting key '%s' to '%v' in %s",
@@ -64,3 +83,14 @@ func (c *keyClient) Delete(path string) error {
 	}
 	return nil
 }
+
+func (c *keyClient) DeleteUnderPrefix(pathPrefix string) error {
+	log.Printf(
+		"[DEBUG] Deleting all keys under prefix '%s' in %s",
+		pathPrefix, c.wOpts.Datacenter,
+	)
+	if _, err := c.client.DeleteTree(pathPrefix, c.wOpts); err != nil {
+		return fmt.Errorf("Failed to delete Consul keys under '%s': %s", pathPrefix, err)
+	}
+	return nil
+}

builtin/providers/consul/resource_consul_key_prefix.go

@@ -0,0 +1,221 @@
+package consul
+
+import (
+	"fmt"
+
+	consulapi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/terraform/helper/schema"
+)
+
+func resourceConsulKeyPrefix() *schema.Resource {
+	return &schema.Resource{
+		Create: resourceConsulKeyPrefixCreate,
+		Update: resourceConsulKeyPrefixUpdate,
+		Read:   resourceConsulKeyPrefixRead,
+		Delete: resourceConsulKeyPrefixDelete,
+
+		Schema: map[string]*schema.Schema{
+			"datacenter": &schema.Schema{
+				Type:     schema.TypeString,
+				Optional: true,
+				Computed: true,
+				ForceNew: true,
+			},
+
+			"token": &schema.Schema{
+				Type:     schema.TypeString,
+				Optional: true,
+			},
+
+			"path_prefix": &schema.Schema{
+				Type:     schema.TypeString,
+				Required: true,
+				ForceNew: true,
+			},
+
+			"subkeys": &schema.Schema{
+				Type:     schema.TypeMap,
+				Required: true,
+				Elem: &schema.Schema{
+					Type: schema.TypeString,
+				},
+			},
+		},
+	}
+}
+
+func resourceConsulKeyPrefixCreate(d *schema.ResourceData, meta interface{}) error {
+	client := meta.(*consulapi.Client)
+	kv := client.KV()
+	token := d.Get("token").(string)
+	dc, err := getDC(d, client)
+	if err != nil {
+		return err
+	}
+
+	keyClient := newKeyClient(kv, dc, token)
+
+	pathPrefix := d.Get("path_prefix").(string)
+	subKeys := map[string]string{}
+	for k, vI := range d.Get("subkeys").(map[string]interface{}) {
+		subKeys[k] = vI.(string)
+	}
+
+	// To reduce the impact of mistakes, we will only "create" a prefix that
+	// is currently empty. This way we are less likely to accidentally
+	// conflict with other mechanisms managing the same prefix.
+	currentSubKeys, err := keyClient.GetUnderPrefix(pathPrefix)
+	if err != nil {
+		return err
+	}
+	if len(currentSubKeys) > 0 {
+		return fmt.Errorf(
+			"%d keys already exist under %s; delete them before managing this prefix with Terraform",
+			len(currentSubKeys), pathPrefix,
+		)
+	}
+
+	// Ideally we'd use d.Partial(true) here so we can correctly record
+	// a partial write, but that mechanism doesn't work for individual map
+	// members, so we record that the resource was created before we
+	// do anything and that way we can recover from errors by doing an
+	// Update on subsequent runs, rather than re-attempting Create with
+	// some keys possibly already present.
+	d.SetId(pathPrefix)
+
+	// Store the datacenter on this resource, which can be helpful for reference
+	// in case it was read from the provider
+	d.Set("datacenter", dc)
+
+	// Now we can just write in all the initial values, since we can expect
+	// that nothing should need deleting yet, as long as there isn't some
+	// other program racing us to write values... which we'll catch on a
+	// subsequent Read.
+	for k, v := range subKeys {
+		fullPath := pathPrefix + k
+		err := keyClient.Put(fullPath, v)
+		if err != nil {
+			return fmt.Errorf("error while writing %s: %s", fullPath, err)
+		}
+	}
+
+	return nil
+}
+
+func resourceConsulKeyPrefixUpdate(d *schema.ResourceData, meta interface{}) error {
+	client := meta.(*consulapi.Client)
+	kv := client.KV()
+	token := d.Get("token").(string)
+	dc, err := getDC(d, client)
+	if err != nil {
+		return err
+	}
+
+	keyClient := newKeyClient(kv, dc, token)
+
+	pathPrefix := d.Id()
+
+	if d.HasChange("subkeys") {
+		o, n := d.GetChange("subkeys")
+		if o == nil {
+			o = map[string]interface{}{}
+		}
+		if n == nil {
+			n = map[string]interface{}{}
+		}
+
+		om := o.(map[string]interface{})
+		nm := n.(map[string]interface{})
+
+		// First we'll write all of the stuff in the "new map" nm,
+		// and then we'll delete any keys that appear in the "old map" om
+		// and do not also appear in nm. This ordering means that if a subkey
+		// name is changed we will briefly have both the old and new names in
+		// Consul, as opposed to briefly having neither.
+
+		// Again, we'd ideally use d.Partial(true) here but it doesn't work
+		// for maps and so we'll just rely on a subsequent Read to tidy up
+		// after a partial write.
+
+		// Write new and changed keys
+		for k, vI := range nm {
+			v := vI.(string)
+			fullPath := pathPrefix + k
+			err := keyClient.Put(fullPath, v)
+			if err != nil {
+				return fmt.Errorf("error while writing %s: %s", fullPath, err)
+			}
+		}
+
+		// Remove deleted keys
+		for k, _ := range om {
+			if _, exists := nm[k]; exists {
+				continue
+			}
+			fullPath := pathPrefix + k
+			err := keyClient.Delete(fullPath)
+			if err != nil {
+				return fmt.Errorf("error while deleting %s: %s", fullPath, err)
+			}
+		}
+
+	}
+
+	// Store the datacenter on this resource, which can be helpful for reference
+	// in case it was read from the provider
+	d.Set("datacenter", dc)
+
+	return nil
+}
+
+func resourceConsulKeyPrefixRead(d *schema.ResourceData, meta interface{}) error {
+	client := meta.(*consulapi.Client)
+	kv := client.KV()
+	token := d.Get("token").(string)
+	dc, err := getDC(d, client)
+	if err != nil {
+		return err
+	}
+
+	keyClient := newKeyClient(kv, dc, token)
+
+	pathPrefix := d.Id()
+
+	subKeys, err := keyClient.GetUnderPrefix(pathPrefix)
+	if err != nil {
+		return err
+	}
+
+	d.Set("subkeys", subKeys)
+
+	// Store the datacenter on this resource, which can be helpful for reference
+	// in case it was read from the provider
+	d.Set("datacenter", dc)
+
+	return nil
+}
+
+func resourceConsulKeyPrefixDelete(d *schema.ResourceData, meta interface{}) error {
+	client := meta.(*consulapi.Client)
+	kv := client.KV()
+	token := d.Get("token").(string)
+	dc, err := getDC(d, client)
+	if err != nil {
+		return err
+	}
+
+	keyClient := newKeyClient(kv, dc, token)
+
+	pathPrefix := d.Id()
+
+	// Delete everything under our prefix, since the entire set of keys under
+	// the given prefix is considered to be managed exclusively by Terraform.
+	err = keyClient.DeleteUnderPrefix(pathPrefix)
+	if err != nil {
+		return err
+	}
+
+	d.SetId("")
+
+	return nil
+}

builtin/providers/consul/resource_consul_key_prefix_test.go

@@ -0,0 +1,150 @@
+package consul
+
+import (
+	"fmt"
+	"testing"
+
+	consulapi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/terraform/helper/resource"
+	"github.com/hashicorp/terraform/terraform"
+)
+
+func TestAccConsulKeyPrefix_basic(t *testing.T) {
+	resource.Test(t, resource.TestCase{
+		PreCheck:  func() { testAccPreCheck(t) },
+		Providers: testAccProviders,
+		CheckDestroy: resource.ComposeTestCheckFunc(
+			testAccCheckConsulKeyPrefixKeyAbsent("species"),
+			testAccCheckConsulKeyPrefixKeyAbsent("meat"),
+			testAccCheckConsulKeyPrefixKeyAbsent("cheese"),
+			testAccCheckConsulKeyPrefixKeyAbsent("bread"),
+		),
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: testAccConsulKeyPrefixConfig,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckConsulKeyPrefixKeyValue("cheese", "chevre"),
+					testAccCheckConsulKeyPrefixKeyValue("bread", "baguette"),
+					testAccCheckConsulKeyPrefixKeyAbsent("species"),
+					testAccCheckConsulKeyPrefixKeyAbsent("meat"),
+				),
+			},
+			resource.TestStep{
+				Config:             testAccConsulKeyPrefixConfig,
+				ExpectNonEmptyPlan: true,
+				Check: resource.ComposeTestCheckFunc(
+					// This will add a rogue key that Terraform isn't
+					// expecting, causing a non-empty plan that wants
+					// to remove it.
+					testAccAddConsulKeyPrefixRogue("species", "gorilla"),
+				),
+			},
+			resource.TestStep{
+				Config: testAccConsulKeyPrefixConfig_Update,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckConsulKeyPrefixKeyValue("meat", "ham"),
+					testAccCheckConsulKeyPrefixKeyValue("bread", "batard"),
+					testAccCheckConsulKeyPrefixKeyAbsent("cheese"),
+					testAccCheckConsulKeyPrefixKeyAbsent("species"),
+				),
+			},
+			resource.TestStep{
+				Config:             testAccConsulKeyPrefixConfig_Update,
+				ExpectNonEmptyPlan: true,
+				Check: resource.ComposeTestCheckFunc(
+					testAccAddConsulKeyPrefixRogue("species", "gorilla"),
+				),
+			},
+		},
+	})
+}
+
+func testAccCheckConsulKeyPrefixDestroy(s *terraform.State) error {
+	kv := testAccProvider.Meta().(*consulapi.Client).KV()
+	opts := &consulapi.QueryOptions{Datacenter: "dc1"}
+	pair, _, err := kv.Get("test/set", opts)
+	if err != nil {
+		return err
+	}
+	if pair != nil {
+		return fmt.Errorf("Key still exists: %#v", pair)
+	}
+	return nil
+}
+
+func testAccCheckConsulKeyPrefixKeyAbsent(name string) resource.TestCheckFunc {
+	fullName := "prefix_test/" + name
+	return func(s *terraform.State) error {
+		kv := testAccProvider.Meta().(*consulapi.Client).KV()
+		opts := &consulapi.QueryOptions{Datacenter: "dc1"}
+		pair, _, err := kv.Get(fullName, opts)
+		if err != nil {
+			return err
+		}
+		if pair != nil {
+			return fmt.Errorf("key '%s' exists, but shouldn't", fullName)
+		}
+		return nil
+	}
+}
+
+// This one is actually not a check, but rather a mutation step. It writes
+// a value directly into Consul, bypassing our Terraform resource.
+func testAccAddConsulKeyPrefixRogue(name, value string) resource.TestCheckFunc {
+	fullName := "prefix_test/" + name
+	return func(s *terraform.State) error {
+		kv := testAccProvider.Meta().(*consulapi.Client).KV()
+		opts := &consulapi.WriteOptions{Datacenter: "dc1"}
+		pair := &consulapi.KVPair{
+			Key:   fullName,
+			Value: []byte(value),
+		}
+		_, err := kv.Put(pair, opts)
+		return err
+	}
+}
+
+func testAccCheckConsulKeyPrefixKeyValue(name, value string) resource.TestCheckFunc {
+	fullName := "prefix_test/" + name
+	return func(s *terraform.State) error {
+		kv := testAccProvider.Meta().(*consulapi.Client).KV()
+		opts := &consulapi.QueryOptions{Datacenter: "dc1"}
+		pair, _, err := kv.Get(fullName, opts)
+		if err != nil {
+			return err
+		}
+		if pair == nil {
+			return fmt.Errorf("key %v doesn't exist, but should", fullName)
+		}
+		if string(pair.Value) != value {
+			return fmt.Errorf("key %v has value %v; want %v", fullName, pair.Value, value)
+		}
+		return nil
+	}
+}
+
+const testAccConsulKeyPrefixConfig = `
+resource "consul_key_prefix" "app" {
+	datacenter = "dc1"
+
+    path_prefix = "prefix_test/"
+
+    subkeys = {
+        cheese = "chevre"
+        bread = "baguette"
+    }
+}
+`
+
+const testAccConsulKeyPrefixConfig_Update = `
+resource "consul_key_prefix" "app" {
+	datacenter = "dc1"
+
+    path_prefix = "prefix_test/"
+
+    subkeys = {
+        bread = "batard"
+        meat = "ham"
+    }
+}
+`

builtin/providers/consul/resource_provider.go

@@ -30,6 +30,7 @@ func Provider() terraform.ResourceProvider {
 
 		ResourcesMap: map[string]*schema.Resource{
 			"consul_keys":       resourceConsulKeys(),
+			"consul_key_prefix": resourceConsulKeyPrefix(),
 		},
 
 		ConfigureFunc: providerConfigure,

website/source/docs/providers/consul/r/key_prefix.html.markdown

@@ -0,0 +1,79 @@
+---
+layout: "consul"
+page_title: "Consul: consul_key_prefix"
+sidebar_current: "docs-consul-resource-key-prefix"
+description: |-
+  Allows Terraform to manage a namespace of Consul keys that share a
+  common name prefix.
+---
+
+# consul\_key\_prefix
+
+Allows Terraform to manage a "namespace" of Consul keys that share a common
+name prefix.
+
+Like `consul_keys`, this resource can write values into the Consul key/value
+store, but *unlike* `consul_keys` this resource can detect and remove extra
+keys that have been added some other way, thus ensuring that rogue data
+added outside of Terraform will be removed on the next run.
+
+This resource is thus useful in the case where Terraform is exclusively
+managing a set of related keys.
+
+To avoid accidentally clobbering matching data that existed in Consul before
+a `consul_key_prefix` resource was created, creation of a key prefix instance
+will fail if any matching keys are already present in the key/value store.
+If any conflicting data is present, you must first delete it manually.
+
+~> **Warning** After this resource is instantiated, Terraform takes control
+over *all* keys with the given path prefix, and will remove any matching keys
+that are not present in the configuration. It will also delete *all* keys under
+the given prefix when a `consul_key_prefix` resource is destroyed, even if
+those keys were created outside of Terraform.
+
+## Example Usage
+
+```
+resource "consul_key_prefix" "myapp_config" {
+    datacenter = "nyc1"
+    token = "abcd"
+
+    # Prefix to add to prepend to all of the subkey names below.
+    path_prefix = "myapp/config/"
+
+    subkeys = {
+        "elb_cname" = "${aws_elb.app.dns_name}"
+        "s3_bucket_name" = "${aws_s3_bucket.app.bucket}"
+        "database/hostname" = "${aws_db_instance.app.address}"
+        "database/port" = "${aws_db_instance.app.port}"
+        "database/username" = "${aws_db_instance.app.username}"
+        "database/password" = "${aws_db_instance.app.password}"
+        "database/name" = "${aws_db_instance.app.name}"
+    }
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `datacenter` - (Optional) The datacenter to use. This overrides the
+  datacenter in the provider setup and the agent's default datacenter.
+
+* `token` - (Optional) The ACL token to use. This overrides the
+  token that the agent provides by default.
+
+* `path_prefix` - (Required) Specifies the common prefix shared by all keys
+  that will be managed by this resource instance. In most cases this will
+  end with a slash, to manage a "folder" of keys.
+
+* `subkeys` - (Required) A mapping from subkey name (which will be appended
+  to the give `path_prefix`) to the value that should be stored at that key.
+  Use slashes as shown in the above example to create "sub-folders" under
+  the given path prefix.
+
+## Attributes Reference
+
+The following attributes are exported:
+
+* `datacenter` - The datacenter the keys are being read/written to.

website/source/docs/providers/consul/r/keys.html.markdown

@@ -13,6 +13,13 @@ to both read keys from Consul, but also to set the value of keys
 in Consul. This is a powerful way dynamically set values in templates,
 and to expose infrastructure details to clients.
 
+This resource manages individual keys, and thus it can create, update and
+delete the keys explicitly given. Howver, It is not able to detect and remove
+additional keys that have been added by non-Terraform means. To manage
+*all* keys sharing a common prefix, and thus have Terraform remove errant keys
+not present in the configuration, consider using the `consul_key_prefix`
+resource instead.
+
 ## Example Usage
 
 ```