From e098c7c24ab5c7e2a3e2269d29b512654b426e70 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoffer=20Kylv=C3=A5g?=
 <christoffer.kylvag@bonniernews.se>
Date: Thu, 9 Mar 2017 08:00:19 +0100
Subject: [PATCH 1/2] backend/remote-state/consul: Add gzip support to consul
 backend client (#8491)

---
 backend/remote-state/consul/client.go | 55 ++++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/backend/remote-state/consul/client.go b/backend/remote-state/consul/client.go
index 8a26165e8..baa3a5c32 100644
--- a/backend/remote-state/consul/client.go
+++ b/backend/remote-state/consul/client.go
@@ -1,6 +1,8 @@
 package consul
 
 import (
+	"bytes"
+	"compress/gzip"
 	"crypto/md5"
 	"encoding/json"
 	"errors"
@@ -16,6 +18,7 @@ import (
 const (
 	lockSuffix     = "/.lock"
 	lockInfoSuffix = "/.lockinfo"
+	maxKVSize      = 512 * 1024
 )
 
 // RemoteClient is a remote client that stores data in Consul.
@@ -36,18 +39,38 @@ func (c *RemoteClient) Get() (*remote.Payload, error) {
 		return nil, nil
 	}
 
+	payload := pair.Value
+	// If the payload starts with 0x1f, it's gzip, not json
+	if len(pair.Value) >= 1 && pair.Value[0] == '\x1f' {
+		if data, err := uncompressState(pair.Value); err == nil {
+			payload = data
+		} else {
+			return nil, err
+		}
+	}
+
 	md5 := md5.Sum(pair.Value)
 	return &remote.Payload{
-		Data: pair.Value,
+		Data: payload,
 		MD5:  md5[:],
 	}, nil
 }
 
 func (c *RemoteClient) Put(data []byte) error {
+	payload := data
+	// If the payload to be written exceeds the Consul KV byte limit, compress
+	if len(data) > maxKVSize {
+		if compressedState, err := compressState(data); err == nil {
+			payload = compressedState
+		} else {
+			return err
+		}
+	}
+
 	kv := c.Client.KV()
 	_, err := kv.Put(&consulapi.KVPair{
 		Key:   c.Path,
-		Value: data,
+		Value: payload,
 	}, nil)
 	return err
 }
@@ -177,3 +200,31 @@ func (c *RemoteClient) Unlock(id string) error {
 
 	return err
 }
+
+func compressState(data []byte) ([]byte, error) {
+	b := new(bytes.Buffer)
+	gz := gzip.NewWriter(b)
+	if _, err := gz.Write(data); err != nil {
+		return nil, err
+	}
+	if err := gz.Flush(); err != nil {
+		return nil, err
+	}
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func uncompressState(data []byte) ([]byte, error) {
+	b := new(bytes.Buffer)
+	gz, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	b.ReadFrom(gz)
+	if err := gz.Close(); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}

From abfa35db7ca4386da73645c16ebe2aed3dcbacfe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoffer=20Kylv=C3=A5g?=
 <christoffer.kylvag@bonniernews.se>
Date: Mon, 13 Mar 2017 08:17:33 +0100
Subject: [PATCH 2/2] backend/remote-state/consul: Make gzip compression
 configurable (#8491)

---
 backend/remote-state/consul/backend.go            | 7 +++++++
 backend/remote-state/consul/backend_state.go      | 4 ++++
 backend/remote-state/consul/client.go             | 5 ++---
 website/source/docs/backends/types/consul.html.md | 1 +
 4 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/backend/remote-state/consul/backend.go b/backend/remote-state/consul/backend.go
index 7193bf8a9..979dc368c 100644
--- a/backend/remote-state/consul/backend.go
+++ b/backend/remote-state/consul/backend.go
@@ -53,6 +53,13 @@ func New() backend.Backend {
 				Description: "HTTP Auth in the format of 'username:password'",
 				Default:     "", // To prevent input
 			},
+
+			"gzip": &schema.Schema{
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Description: "Compress the state data using gzip",
+				Default:     false,
+			},
 		},
 	}
 
diff --git a/backend/remote-state/consul/backend_state.go b/backend/remote-state/consul/backend_state.go
index 6e6d115f0..0339dd79a 100644
--- a/backend/remote-state/consul/backend_state.go
+++ b/backend/remote-state/consul/backend_state.go
@@ -85,11 +85,15 @@ func (b *Backend) State(name string) (state.State, error) {
 	// Determine the path of the data
 	path := b.path(name)
 
+	// Determine whether to gzip or not
+	gzip := b.configData.Get("gzip").(bool)
+
 	// Build the state client
 	stateMgr := &remote.State{
 		Client: &RemoteClient{
 			Client: client,
 			Path:   path,
+			GZip:   gzip,
 		},
 	}
 
diff --git a/backend/remote-state/consul/client.go b/backend/remote-state/consul/client.go
index baa3a5c32..cd5971163 100644
--- a/backend/remote-state/consul/client.go
+++ b/backend/remote-state/consul/client.go
@@ -18,13 +18,13 @@ import (
 const (
 	lockSuffix     = "/.lock"
 	lockInfoSuffix = "/.lockinfo"
-	maxKVSize      = 512 * 1024
 )
 
 // RemoteClient is a remote client that stores data in Consul.
 type RemoteClient struct {
 	Client *consulapi.Client
 	Path   string
+	GZip   bool
 
 	consulLock *consulapi.Lock
 	lockCh     <-chan struct{}
@@ -58,8 +58,7 @@ func (c *RemoteClient) Get() (*remote.Payload, error) {
 
 func (c *RemoteClient) Put(data []byte) error {
 	payload := data
-	// If the payload to be written exceeds the Consul KV byte limit, compress
-	if len(data) > maxKVSize {
+	if c.GZip {
 		if compressedState, err := compressState(data); err == nil {
 			payload = compressedState
 		} else {
diff --git a/website/source/docs/backends/types/consul.html.md b/website/source/docs/backends/types/consul.html.md
index 59c05724e..24d45fe14 100644
--- a/website/source/docs/backends/types/consul.html.md
+++ b/website/source/docs/backends/types/consul.html.md
@@ -53,3 +53,4 @@ The following configuration options / environment variables are supported:
  * `datacenter` - (Optional) The datacenter to use. Defaults to that of the agent.
  * `http_auth` / `CONSUL_HTTP_AUTH` - (Optional) HTTP Basic Authentication credentials to be used when
    communicating with Consul, in the format of either `user` or `user:pass`.
+ * `gzip` - (Optional) `true` to compress the state data using gzip, or `false` (the default) to leave it uncompressed.