Add google_storage_signed_url data source.

This commit is contained in:
Matt Morrison 2016-08-12 14:52:44 +12:00 committed by Matt Morrison
parent 19e8932a92
commit 80a42feb5a
3 changed files with 513 additions and 1 deletions

View File

@ -0,0 +1,299 @@
package google
import (
"bytes"
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"github.com/hashicorp/terraform/helper/pathorcontents"
"github.com/hashicorp/terraform/helper/schema"
"golang.org/x/oauth2/google"
"golang.org/x/oauth2/jwt"
"log"
"net/url"
"os"
"os/user"
"strconv"
"strings"
"time"
)
const gcsBaseUrl = "https://storage.googleapis.com"
const envVar = "GOOGLE_APPLICATION_CREDENTIALS"
func dataSourceGoogleSignedUrl() *schema.Resource {
return &schema.Resource{
Read: dataSourceGoogleSignedUrlRead,
Schema: map[string]*schema.Schema{
"bucket": &schema.Schema{
Type: schema.TypeString,
Required: true,
},
//TODO: implement support
//"content_type": &schema.Schema{
// Type: schema.TypeString,
// Optional: true,
// Default: "",
//},
"credentials": &schema.Schema{
Type: schema.TypeString,
Optional: true,
},
"duration": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Default: "1h",
},
"http_method": &schema.Schema{
Type: schema.TypeString,
Optional: true,
Default: "GET",
},
//TODO: implement support
//"http_headers": &schema.Schema{
// Type: schema.TypeList,
// Optional: true,
//},
//TODO: implement support
//"md5_digest": &schema.Schema{
// Type: schema.TypeString,
// Optional: true,
// Default: "",
//},
"path": &schema.Schema{
Type: schema.TypeString,
Required: true,
},
"signed_url": &schema.Schema{
Type: schema.TypeString,
Computed: true,
},
},
}
}
func dataSourceGoogleSignedUrlRead(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Build UrlData object from data source attributes
urlData := &UrlData{}
// HTTP Method
if method, ok := d.GetOk("http_method"); ok && len(method.(string)) >= 3 {
urlData.HttpMethod = method.(string)
} else {
return fmt.Errorf("not a valid http method")
}
// convert duration to an expiration datetime (unix time in seconds)
durationString := "1h"
if v, ok := d.GetOk("duration"); ok {
durationString = v.(string)
}
duration, err := time.ParseDuration(durationString)
if err != nil {
return fmt.Errorf("could not parse duration")
}
expires := time.Now().Unix() + int64(duration.Seconds())
urlData.Expires = int(expires)
// object path
path := []string{
"",
d.Get("bucket").(string),
d.Get("path").(string),
}
objectPath := strings.Join(path, "/")
urlData.Path = objectPath
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Load JWT Config from Google Credentials
jwtConfig, err := loadJwtConfig(d, config)
if err != nil {
return err
}
urlData.JwtConfig = jwtConfig
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Sign url object data
signature, err := SignString(urlData.CreateSigningString(), jwtConfig)
if err != nil {
return fmt.Errorf("could not sign data: %v", err)
}
urlData.Signature = signature
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// Construct URL
finalUrl := urlData.BuildUrl()
d.SetId(finalUrl)
d.Set("signed_url", finalUrl)
return nil
}
// This looks for credentials json in the following places,
// preferring the first location found:
//
// 1. Credentials provided in data source `credentials` attribute.
// 2. Credentials provided in the provider definition.
// 3. A JSON file whose path is specified by the
// GOOGLE_APPLICATION_CREDENTIALS environment variable.
func loadJwtConfig(d *schema.ResourceData, meta interface{}) (*jwt.Config, error) {
config := meta.(*Config)
credentials := ""
if v, ok := d.GetOk("credentials"); ok {
log.Println("[DEBUG] using data source credentials")
credentials = v.(string)
} else if config.Credentials != "" {
log.Println("[DEBUG] using provider credentials")
credentials = config.Credentials
} else if filename := os.Getenv(envVar); filename != "" {
log.Println("[DEBUG] using env GOOGLE_APPLICATION_CREDENTIALS credentials")
credentials = filename
}
if strings.TrimSpace(credentials) != "" {
contents, _, err := pathorcontents.Read(credentials)
if err != nil {
return nil, fmt.Errorf("Error loading credentials: %s", err)
}
cfg, err := google.JWTConfigFromJSON([]byte(contents), "")
if err != nil {
return nil, fmt.Errorf("Error parsing credentials: \n %s \n Error: %s", contents, err)
}
return cfg, nil
}
return nil, fmt.Errorf("Credentials not provided in resource or provider configuration or GOOGLE_APPLICATION_CREDENTIALS environment variable.")
}
func guessUnixHomeDir() string {
usr, err := user.Current()
if err == nil {
return usr.HomeDir
}
return os.Getenv("HOME")
}
// parsePrivateKey converts the binary contents of a private key file
// to an *rsa.PrivateKey. It detects whether the private key is in a
// PEM container or not. If so, it extracts the the private key
// from PEM container before conversion. It only supports PEM
// containers with no passphrase.
// copied from golang.org/x/oauth2/internal
func parsePrivateKey(key []byte) (*rsa.PrivateKey, error) {
block, _ := pem.Decode(key)
if block != nil {
key = block.Bytes
}
parsedKey, err := x509.ParsePKCS8PrivateKey(key)
if err != nil {
parsedKey, err = x509.ParsePKCS1PrivateKey(key)
if err != nil {
return nil, fmt.Errorf("private key should be a PEM or plain PKSC1 or PKCS8; parse error: %v", err)
}
}
parsed, ok := parsedKey.(*rsa.PrivateKey)
if !ok {
return nil, fmt.Errorf("private key is invalid")
}
return parsed, nil
}
type UrlData struct {
JwtConfig *jwt.Config
HttpMethod string
Expires int
Path string
Signature []byte
}
// Creates a string in the form ready for signing:
// https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
// Example output:
// -------------------
// GET
//
//
// 1388534400
// bucket/objectname
// -------------------
func (u *UrlData) CreateSigningString() []byte {
var buf bytes.Buffer
// HTTP VERB
buf.WriteString(u.HttpMethod)
buf.WriteString("\n")
// MD5 digest (optional)
// TODO
buf.WriteString("\n")
// request content-type (optional)
// TODO
buf.WriteString("\n")
// signed url expiration
buf.WriteString(strconv.Itoa(u.Expires))
buf.WriteString("\n")
// additional request headers (optional)
// TODO
// object path
buf.WriteString(u.Path)
return buf.Bytes()
}
// Builds the final signed URL a client can use to retrieve storage object
func (u *UrlData) BuildUrl() string {
// base64 encode signature
encoded := base64.StdEncoding.EncodeToString(u.Signature)
// encoded signature may include /, = characters that need escaping
encoded = url.QueryEscape(encoded)
// set url
// https://cloud.google.com/storage/docs/access-control/create-signed-urls-program
var urlBuffer bytes.Buffer
urlBuffer.WriteString(gcsBaseUrl)
urlBuffer.WriteString(u.Path)
urlBuffer.WriteString("?GoogleAccessId=")
urlBuffer.WriteString(u.JwtConfig.Email)
urlBuffer.WriteString("&Expires=")
urlBuffer.WriteString(strconv.Itoa(u.Expires))
urlBuffer.WriteString("&Signature=")
urlBuffer.WriteString(encoded)
return urlBuffer.String()
}
func SignString(toSign []byte, cfg *jwt.Config) ([]byte, error) {
pk, err := parsePrivateKey(cfg.PrivateKey)
if err != nil {
return nil, fmt.Errorf("could not parse key: %v\nKey:%s", err, string(cfg.PrivateKey))
}
// Hash string
hasher := sha256.New()
hasher.Write(toSign)
signed, err := rsa.SignPKCS1v15(rand.Reader, pk, crypto.SHA256, hasher.Sum(nil))
if err != nil {
return nil, fmt.Errorf("Error from signing: %s\n", err)
}
return signed, nil
}

View File

@ -0,0 +1,212 @@
package google
import (
"testing"
"bytes"
"encoding/base64"
"fmt"
"github.com/hashicorp/terraform/helper/acctest"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/terraform"
"golang.org/x/oauth2/google"
"io/ioutil"
"net/http"
"net/url"
)
const fakeCredentials = `{
"type": "service_account",
"project_id": "gcp-project",
"private_key_id": "29a54056cee3d6886d9e8515a959af538ab5add9",
"private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEAsGHDAdHZfi81LgVeeMHXYLgNDpcFYhoBykYtTDdNyA5AixID\n8JdKlCmZ6qLNnZrbs4JlBJfmzw6rjUC5bVBFg5NwYVBu3+3Msa4rgLsTGsjPH9rt\nC+QFnFhcmzg3zz8eeXBqJdhw7wmn1Xa9SsC3h6YWveBk98ecyE7yGe8J8xGphjk7\nEQ/KBmRK/EJD0ZwuYW1W4Bv5f5fca7qvi9rCprEmL8//uy0qCwoJj2jU3zc5p72M\npkSZb1XlYxxTEo/h9WCEvWS9pGhy6fJ0sA2RsBHqU4Y5O7MJEei9yu5fVSZUi05f\n/ggfUID+cFEq0Z/A98whKPEBBJ/STdEaqEEkBwIDAQABAoIBAED6EsvF0dihbXbh\ntXbI+h4AT5cTXYFRUV2B0sgkC3xqe65/2YG1Sl0gojoE9bhcxxjvLWWuy/F1Vw93\nS5gQnTsmgpzm86F8yg6euhn3UMdqOJtknDToMITzLFJmOHEZsJFOL1x3ysrUhMan\nsn4qVrIbJn+WfbumBoToSFnzbHflacOh06ZRbYa2bpSPMfGGFtwqQjRadn5+pync\nlCjaupcg209sM0qEk/BDSzHvWL1VgLMdiKBx574TSwS0o569+7vPNt92Ydi7kARo\nreOzkkF4L3xNhKZnmls2eGH6A8cp1KZXoMLFuO+IwvBMA0O29LsUlKJU4PjBrf+7\nwaslnMECgYEA5bJv0L6DKZQD3RCBLue4/mDg0GHZqAhJBS6IcaXeaWeH6PgGZggV\nMGkWnULltJIYFwtaueTfjWqciAeocKx+rqoRjuDMOGgcrEf6Y+b5AqF+IjQM66Ll\nIYPUt3FCIc69z5LNEtyP4DSWsFPJ5UhAoG4QRlDTqT5q0gKHFjeLdeECgYEAxJRk\nkrsWmdmUs5NH9pyhTdEDIc59EuJ8iOqOLzU8xUw6/s2GSClopEFJeeEoIWhLuPY3\nX3bFt4ppl/ksLh05thRs4wXRxqhnokjD3IcGu3l6Gb5QZTYwb0VfN+q2tWVEE8Qc\nPQURheUsM2aP/gpJVQvNsWVmkT0Ijc3J8bR2hucCgYEAjOF4e0ueHu5NwFTTJvWx\nHTRGLwkU+l66ipcT0MCvPW7miRk2s3XZqSuLV0Ekqi/A3sF0D/g0tQPipfwsb48c\n0/wzcLKoDyCsFW7AQG315IswVcIe+peaeYfl++1XZmzrNlkPtrXY+ObIVbXOavZ5\nzOw0xyvj5jYGRnCOci33N4ECgYA91EKx2ABq0YGw3aEj0u31MMlgZ7b1KqFq2wNv\nm7oKgEiJ/hC/P673AsXefNAHeetfOKn/77aOXQ2LTEb2FiEhwNjiquDpL+ywoVxh\nT2LxsmqSEEbvHpUrWlFxn/Rpp3k7ElKjaqWxTHyTii2+BHQ+OKEwq6kQA3deSpy6\n1jz1fwKBgQDLqbdq5FA63PWqApfNVykXukg9MASIcg/0fjADFaHTPDvJjhFutxRP\nppI5Q95P12CQ/eRBZKJnRlkhkL8tfPaWPzzOpCTjID7avRhx2oLmstmYuXx0HluE\ncqXLbAV9WDpIJ3Bpa/S8tWujWhLDmixn2JeAdurWS+naH9U9e4I6Rw==\n-----END RSA PRIVATE KEY-----\n",
"client_email": "user@gcp-project.iam.gserviceaccount.com",
"client_id": "103198861025845558729",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/user%40gcp-project.iam.gserviceaccount.com"
}`
// The following values are derived from the output of the `gsutil signurl` command.
// i.e.
// gsutil signurl fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
// URL HTTP Method Expiration Signed URL
// gs://tf-test-bucket-6159205297736845881/path/to/file GET 2016-08-12 14:03:30 https://storage.googleapis.com/tf-test-bucket-6159205297736845881/path/to/file?GoogleAccessId=user@gcp-project.iam.gserviceaccount.com&Expires=1470967410&Signature=JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D
const testUrlPath = "/tf-test-bucket-6159205297736845881/path/to/file"
const testUrlExpires = 1470967410
const testUrlExpectedSignatureBase64Encoded = "JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D"
const testUrlExpectedUrl = "https://storage.googleapis.com/tf-test-bucket-6159205297736845881/path/to/file?GoogleAccessId=user@gcp-project.iam.gserviceaccount.com&Expires=1470967410&Signature=JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D"
func TestUrlData_Signing(t *testing.T) {
urlData := &UrlData{
HttpMethod: "GET",
Expires: testUrlExpires,
Path: testUrlPath,
}
// unescape and decode the expected signature
expectedSig, err := url.QueryUnescape(testUrlExpectedSignatureBase64Encoded)
if err != nil {
t.Error(err)
}
expected, err := base64.StdEncoding.DecodeString(expectedSig)
if err != nil {
t.Error(err)
}
// load fake service account credentials
cfg, err := google.JWTConfigFromJSON([]byte(fakeCredentials), "")
if err != nil {
t.Error(err)
}
// create url data signature
toSign := urlData.CreateSigningString()
result, err := SignString(toSign, cfg)
if err != nil {
t.Error(err)
}
// compare to expected value
if !bytes.Equal(result, expected) {
t.Errorf("Signatures do not match:\n%x\n%x\n", expected, result)
}
}
func TestUrlData_CreateUrl(t *testing.T) {
// unescape and decode the expected signature
encodedSig, err := url.QueryUnescape(testUrlExpectedSignatureBase64Encoded)
if err != nil {
t.Error(err)
}
sig, err := base64.StdEncoding.DecodeString(encodedSig)
if err != nil {
t.Error(err)
}
// load fake service account credentials
cfg, err := google.JWTConfigFromJSON([]byte(fakeCredentials), "")
if err != nil {
t.Error(err)
}
urlData := &UrlData{
HttpMethod: "GET",
Expires: testUrlExpires,
Path: testUrlPath,
Signature: sig,
JwtConfig: cfg,
}
result := urlData.BuildUrl()
if result != testUrlExpectedUrl {
t.Errorf("URL does not match expected value:\n%s\n%s", testUrlExpectedUrl, result)
}
}
func TestDatasourceSignedUrl_basic(t *testing.T) {
resource.Test(t, resource.TestCase{
//PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
resource.TestStep{
Config: testGoogleSignedUrlConfig,
Check: resource.ComposeTestCheckFunc(
testAccGoogleSignedUrlExists("data.google_storage_object_signed_url.blerg"),
),
},
},
})
}
func TestDatasourceSignedUrl_accTest(t *testing.T) {
bucketName := fmt.Sprintf("tf-test-bucket-%d", acctest.RandInt())
resource.Test(t, resource.TestCase{
//PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
Steps: []resource.TestStep{
resource.TestStep{
Config: testAccTestGoogleStorageObjectSingedUrl(bucketName),
Check: resource.ComposeTestCheckFunc(
testAccGoogleSignedUrlRetrieval("data.google_storage_object_signed_url.story_url"),
),
},
},
})
}
func testAccGoogleSignedUrlExists(n string) resource.TestCheckFunc {
return func(s *terraform.State) error {
r := s.RootModule().Resources[n]
a := r.Primary.Attributes
if a["signed_url"] == "" {
return fmt.Errorf("signed_url is empty: %v", a)
}
return nil
}
}
func testAccGoogleSignedUrlRetrieval(n string) resource.TestCheckFunc {
return func(s *terraform.State) error {
r := s.RootModule().Resources[n]
a := r.Primary.Attributes
if a["signed_url"] == "" {
return fmt.Errorf("signed_url is empty: %v", a)
}
url := a["signed_url"]
// send request to GET object using signed url
client := http.DefaultClient
response, err := client.Get(url)
if err != nil {
return err
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
if string(body) != "once upon a time..." {
return fmt.Errorf("Got unexpected object contents: %s\n\tURL: %s", string(body), url)
}
return nil
}
}
const testGoogleSignedUrlConfig = `
data "google_storage_object_signed_url" "blerg" {
bucket = "friedchicken"
path = "path/to/file"
}
`
func testAccTestGoogleStorageObjectSingedUrl(bucketName string) string {
return fmt.Sprintf(`
resource "google_storage_bucket" "bucket" {
name = "%s"
}
resource "google_storage_bucket_object" "story" {
name = "path/to/file"
bucket = "${google_storage_bucket.bucket.name}"
content = "once upon a time..."
}
data "google_storage_object_signed_url" "story_url" {
bucket = "${google_storage_bucket.bucket.name}"
path = "${google_storage_bucket_object.story.name}"
}
`, bucketName)
}

View File

@ -57,7 +57,8 @@ func Provider() terraform.ResourceProvider {
}, },
DataSourcesMap: map[string]*schema.Resource{ DataSourcesMap: map[string]*schema.Resource{
"google_iam_policy": dataSourceGoogleIamPolicy(), "google_iam_policy": dataSourceGoogleIamPolicy(),
"google_storage_object_signed_url": dataSourceGoogleSignedUrl(),
}, },
ResourcesMap: map[string]*schema.Resource{ ResourcesMap: map[string]*schema.Resource{