Merge pull request #4254 from hashicorp/b-aws-auth-refactor

provider/aws: WIP Refactor AWS Authentication chain

builtin/providers/aws/config.go

@@ -3,14 +3,19 @@ package aws
 import (
 	"fmt"
 	"log"
+	"net/http"
+	"os"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-multierror"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
+	awsCredentials "github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/cloudformation"
@@ -104,9 +109,14 @@ func (c *Config) Client() (interface{}, error) {
 		client.region = c.Region
 
 		log.Println("[INFO] Building AWS auth structure")
-		// We fetched all credential sources in Provider. If they are
+		creds := getCreds(c.AccessKey, c.SecretKey, c.Token)
-		// available, they'll already be in c. See Provider definition.
+		// Call Get to check for credential provider. If nothing found, we'll get an
-		creds := credentials.NewStaticCredentials(c.AccessKey, c.SecretKey, c.Token)
+		// error, and we can present it nicely to the user
+		_, err = creds.Get()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("Error loading credentials for AWS Provider: %s", err))
+			return nil, &multierror.Error{Errors: errs}
+		}
 		awsConfig := &aws.Config{
 			Credentials: creds,
 			Region:      aws.String(c.Region),
@@ -118,7 +128,7 @@ func (c *Config) Client() (interface{}, error) {
 		sess := session.New(awsConfig)
 		client.iamconn = iam.New(sess)
 
-		err := c.ValidateCredentials(client.iamconn)
+		err = c.ValidateCredentials(client.iamconn)
 		if err != nil {
 			errs = append(errs, err)
 		}
@@ -316,3 +326,53 @@ func (c *Config) ValidateAccountId(iamconn *iam.IAM) error {
 
 	return nil
 }
+
+// This function is responsible for reading credentials from the
+// environment in the case that they're not explicitly specified
+// in the Terraform configuration.
+func getCreds(key, secret, token string) *awsCredentials.Credentials {
+	// build a chain provider, lazy-evaulated by aws-sdk
+	providers := []awsCredentials.Provider{
+		&awsCredentials.StaticProvider{Value: awsCredentials.Value{
+			AccessKeyID:     key,
+			SecretAccessKey: secret,
+			SessionToken:    token,
+		}},
+		&awsCredentials.EnvProvider{},
+		&awsCredentials.SharedCredentialsProvider{},
+	}
+
+	// We only look in the EC2 metadata API if we can connect
+	// to the metadata service within a reasonable amount of time
+	metadataURL := os.Getenv("AWS_METADATA_URL")
+	if metadataURL == "" {
+		metadataURL = "http://169.254.169.254:80/latest"
+	}
+	c := http.Client{
+		Timeout: 100 * time.Millisecond,
+	}
+
+	r, err := c.Get(metadataURL)
+	// Flag to determine if we should add the EC2Meta data provider. Default false
+	var useIAM bool
+	if err == nil {
+		// AWS will add a "Server: EC2ws" header value for the metadata request. We
+		// check the headers for this value to ensure something else didn't just
+		// happent to be listening on that IP:Port
+		if r.Header["Server"] != nil && strings.Contains(r.Header["Server"][0], "EC2") {
+			useIAM = true
+		}
+	}
+
+	if useIAM {
+		log.Printf("[DEBUG] EC2 Metadata service found, adding EC2 Role Credential Provider")
+		providers = append(providers, &ec2rolecreds.EC2RoleProvider{
+			Client: ec2metadata.New(session.New(&aws.Config{
+				Endpoint: aws.String(metadataURL),
+			})),
+		})
+	} else {
+		log.Printf("[DEBUG] EC2 Metadata service not found, not adding EC2 Role Credential Provider")
+	}
+	return awsCredentials.NewChainCredentials(providers)
+}

builtin/providers/aws/config_test.go

@@ -0,0 +1,299 @@
+package aws
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+)
+
+func TestAWSConfig_shouldError(t *testing.T) {
+	resetEnv := unsetEnv(t)
+	defer resetEnv()
+	cfg := Config{}
+
+	c := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token)
+	_, err := c.Get()
+	if awsErr, ok := err.(awserr.Error); ok {
+		if awsErr.Code() != "NoCredentialProviders" {
+			t.Fatalf("Expected NoCredentialProviders error")
+		}
+	}
+	if err == nil {
+		t.Fatalf("Expected an error with empty env, keys, and IAM in AWS Config")
+	}
+}
+
+func TestAWSConfig_shouldBeStatic(t *testing.T) {
+	simple := []struct {
+		Key, Secret, Token string
+	}{
+		{
+			Key:    "test",
+			Secret: "secret",
+		}, {
+			Key:    "test",
+			Secret: "test",
+			Token:  "test",
+		},
+	}
+
+	for _, c := range simple {
+		cfg := Config{
+			AccessKey: c.Key,
+			SecretKey: c.Secret,
+			Token:     c.Token,
+		}
+
+		creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token)
+		if creds == nil {
+			t.Fatalf("Expected a static creds provider to be returned")
+		}
+		v, err := creds.Get()
+		if err != nil {
+			t.Fatalf("Error gettings creds: %s", err)
+		}
+		if v.AccessKeyID != c.Key {
+			t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", c.Key, v.AccessKeyID)
+		}
+		if v.SecretAccessKey != c.Secret {
+			t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", c.Secret, v.SecretAccessKey)
+		}
+		if v.SessionToken != c.Token {
+			t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", c.Token, v.SessionToken)
+		}
+	}
+}
+
+// TestAWSConfig_shouldIAM is designed to test the scenario of running Terraform
+// from an EC2 instance, without environment variables or manually supplied
+// credentials.
+func TestAWSConfig_shouldIAM(t *testing.T) {
+	// clear AWS_* environment variables
+	resetEnv := unsetEnv(t)
+	defer resetEnv()
+
+	// capture the test server's close method, to call after the test returns
+	ts := awsEnv(t)
+	defer ts()
+
+	// An empty config, no key supplied
+	cfg := Config{}
+
+	creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token)
+	if creds == nil {
+		t.Fatalf("Expected a static creds provider to be returned")
+	}
+
+	v, err := creds.Get()
+	if err != nil {
+		t.Fatalf("Error gettings creds: %s", err)
+	}
+	if v.AccessKeyID != "somekey" {
+		t.Fatalf("AccessKeyID mismatch, expected: (somekey), got (%s)", v.AccessKeyID)
+	}
+	if v.SecretAccessKey != "somesecret" {
+		t.Fatalf("SecretAccessKey mismatch, expected: (somesecret), got (%s)", v.SecretAccessKey)
+	}
+	if v.SessionToken != "sometoken" {
+		t.Fatalf("SessionToken mismatch, expected: (sometoken), got (%s)", v.SessionToken)
+	}
+}
+
+// TestAWSConfig_shouldIAM is designed to test the scenario of running Terraform
+// from an EC2 instance, without environment variables or manually supplied
+// credentials.
+func TestAWSConfig_shouldIgnoreIAM(t *testing.T) {
+	resetEnv := unsetEnv(t)
+	defer resetEnv()
+	// capture the test server's close method, to call after the test returns
+	ts := awsEnv(t)
+	defer ts()
+	simple := []struct {
+		Key, Secret, Token string
+	}{
+		{
+			Key:    "test",
+			Secret: "secret",
+		}, {
+			Key:    "test",
+			Secret: "test",
+			Token:  "test",
+		},
+	}
+
+	for _, c := range simple {
+		cfg := Config{
+			AccessKey: c.Key,
+			SecretKey: c.Secret,
+			Token:     c.Token,
+		}
+
+		creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token)
+		if creds == nil {
+			t.Fatalf("Expected a static creds provider to be returned")
+		}
+		v, err := creds.Get()
+		if err != nil {
+			t.Fatalf("Error gettings creds: %s", err)
+		}
+		if v.AccessKeyID != c.Key {
+			t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", c.Key, v.AccessKeyID)
+		}
+		if v.SecretAccessKey != c.Secret {
+			t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", c.Secret, v.SecretAccessKey)
+		}
+		if v.SessionToken != c.Token {
+			t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", c.Token, v.SessionToken)
+		}
+	}
+}
+
+func TestAWSConfig_shouldBeENV(t *testing.T) {
+	// need to set the environment variables to a dummy string, as we don't know
+	// what they may be at runtime without hardcoding here
+	s := "some_env"
+	resetEnv := setEnv(s, t)
+	defer resetEnv()
+
+	cfg := Config{}
+	creds := getCreds(cfg.AccessKey, cfg.SecretKey, cfg.Token)
+	if creds == nil {
+		t.Fatalf("Expected a static creds provider to be returned")
+	}
+	v, err := creds.Get()
+	if err != nil {
+		t.Fatalf("Error gettings creds: %s", err)
+	}
+	if v.AccessKeyID != s {
+		t.Fatalf("AccessKeyID mismatch, expected: (%s), got (%s)", s, v.AccessKeyID)
+	}
+	if v.SecretAccessKey != s {
+		t.Fatalf("SecretAccessKey mismatch, expected: (%s), got (%s)", s, v.SecretAccessKey)
+	}
+	if v.SessionToken != s {
+		t.Fatalf("SessionToken mismatch, expected: (%s), got (%s)", s, v.SessionToken)
+	}
+}
+
+// unsetEnv unsets enviornment variables for testing a "clean slate" with no
+// credentials in the environment
+func unsetEnv(t *testing.T) func() {
+	// Grab any existing AWS keys and preserve. In some tests we'll unset these, so
+	// we need to have them and restore them after
+	e := getEnv()
+	if err := os.Unsetenv("AWS_ACCESS_KEY_ID"); err != nil {
+		t.Fatalf("Error unsetting env var AWS_ACCESS_KEY_ID: %s", err)
+	}
+	if err := os.Unsetenv("AWS_SECRET_ACCESS_KEY"); err != nil {
+		t.Fatalf("Error unsetting env var AWS_SECRET_ACCESS_KEY: %s", err)
+	}
+	if err := os.Unsetenv("AWS_SESSION_TOKEN"); err != nil {
+		t.Fatalf("Error unsetting env var AWS_SESSION_TOKEN: %s", err)
+	}
+
+	return func() {
+		// re-set all the envs we unset above
+		if err := os.Setenv("AWS_ACCESS_KEY_ID", e.Key); err != nil {
+			t.Fatalf("Error resetting env var AWS_ACCESS_KEY_ID: %s", err)
+		}
+		if err := os.Setenv("AWS_SECRET_ACCESS_KEY", e.Secret); err != nil {
+			t.Fatalf("Error resetting env var AWS_SECRET_ACCESS_KEY: %s", err)
+		}
+		if err := os.Setenv("AWS_SESSION_TOKEN", e.Token); err != nil {
+			t.Fatalf("Error resetting env var AWS_SESSION_TOKEN: %s", err)
+		}
+	}
+}
+
+func setEnv(s string, t *testing.T) func() {
+	e := getEnv()
+	// Set all the envs to a dummy value
+	if err := os.Setenv("AWS_ACCESS_KEY_ID", s); err != nil {
+		t.Fatalf("Error setting env var AWS_ACCESS_KEY_ID: %s", err)
+	}
+	if err := os.Setenv("AWS_SECRET_ACCESS_KEY", s); err != nil {
+		t.Fatalf("Error setting env var AWS_SECRET_ACCESS_KEY: %s", err)
+	}
+	if err := os.Setenv("AWS_SESSION_TOKEN", s); err != nil {
+		t.Fatalf("Error setting env var AWS_SESSION_TOKEN: %s", err)
+	}
+
+	return func() {
+		// re-set all the envs we unset above
+		if err := os.Setenv("AWS_ACCESS_KEY_ID", e.Key); err != nil {
+			t.Fatalf("Error resetting env var AWS_ACCESS_KEY_ID: %s", err)
+		}
+		if err := os.Setenv("AWS_SECRET_ACCESS_KEY", e.Secret); err != nil {
+			t.Fatalf("Error resetting env var AWS_SECRET_ACCESS_KEY: %s", err)
+		}
+		if err := os.Setenv("AWS_SESSION_TOKEN", e.Token); err != nil {
+			t.Fatalf("Error resetting env var AWS_SESSION_TOKEN: %s", err)
+		}
+	}
+}
+
+// awsEnv establishes a httptest server to mock out the internal AWS Metadata
+// service. IAM Credentials are retrieved by the EC2RoleProvider, which makes
+// API calls to this internal URL. By replacing the server with a test server,
+// we can simulate an AWS environment
+func awsEnv(t *testing.T) func() {
+	routes := routes{}
+	if err := json.Unmarshal([]byte(aws_routes), &routes); err != nil {
+		t.Fatalf("Failed to unmarshal JSON in AWS ENV test: %s", err)
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.Header().Add("Server", "MockEC2")
+		for _, e := range routes.Endpoints {
+			if r.RequestURI == e.Uri {
+				fmt.Fprintln(w, e.Body)
+			}
+		}
+	}))
+
+	os.Setenv("AWS_METADATA_URL", ts.URL+"/latest")
+	return ts.Close
+}
+
+func getEnv() *currentEnv {
+	// Grab any existing AWS keys and preserve. In some tests we'll unset these, so
+	// we need to have them and restore them after
+	return &currentEnv{
+		Key:    os.Getenv("AWS_ACCESS_KEY_ID"),
+		Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
+		Token:  os.Getenv("AWS_SESSION_TOKEN"),
+	}
+}
+
+// struct to preserve the current environment
+type currentEnv struct {
+	Key, Secret, Token string
+}
+
+type routes struct {
+	Endpoints []*endpoint `json:"endpoints"`
+}
+type endpoint struct {
+	Uri  string `json:"uri"`
+	Body string `json:"body"`
+}
+
+const aws_routes = `
+{
+  "endpoints": [
+    {
+      "uri": "/latest/meta-data/iam/security-credentials",
+      "body": "test_role"
+    },
+    {
+      "uri": "/latest/meta-data/iam/security-credentials/test_role",
+      "body": "{\"Code\":\"Success\",\"LastUpdated\":\"2015-12-11T17:17:25Z\",\"Type\":\"AWS-HMAC\",\"AccessKeyId\":\"somekey\",\"SecretAccessKey\":\"somesecret\",\"Token\":\"sometoken\"}"
+    }
+  ]
+}
+`

builtin/providers/aws/provider.go

@@ -1,19 +1,10 @@
 package aws
 
 import (
-	"net"
-	"sync"
-	"time"
-
 	"github.com/hashicorp/terraform/helper/hashcode"
 	"github.com/hashicorp/terraform/helper/mutexkv"
 	"github.com/hashicorp/terraform/helper/schema"
 	"github.com/hashicorp/terraform/terraform"
-
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
-	"github.com/aws/aws-sdk-go/aws/ec2metadata"
-	"github.com/aws/aws-sdk-go/aws/session"
 )
 
 // Provider returns a terraform.ResourceProvider.
@@ -21,95 +12,27 @@ func Provider() terraform.ResourceProvider {
 	// TODO: Move the validation to this, requires conditional schemas
 	// TODO: Move the configuration to this, requires validation
 
-	// These variables are closed within the `getCreds` function below.
-	// This function is responsible for reading credentials from the
-	// environment in the case that they're not explicitly specified
-	// in the Terraform configuration.
-	//
-	// By using the getCreds function here instead of making the default
-	// empty, we avoid asking for input on credentials if they're available
-	// in the environment.
-	var credVal credentials.Value
-	var credErr error
-	var once sync.Once
-	getCreds := func() {
-		// Build the list of providers to look for creds in
-		providers := []credentials.Provider{
-			&credentials.EnvProvider{},
-			&credentials.SharedCredentialsProvider{},
-		}
-
-		// We only look in the EC2 metadata API if we can connect
-		// to the metadata service within a reasonable amount of time
-		conn, err := net.DialTimeout("tcp", "169.254.169.254:80", 100*time.Millisecond)
-		if err == nil {
-			conn.Close()
-			providers = append(providers, &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())})
-		}
-
-		credVal, credErr = credentials.NewChainCredentials(providers).Get()
-
-		// If we didn't successfully find any credentials, just
-		// set the error to nil.
-		if credErr == credentials.ErrNoValidProvidersFoundInChain {
-			credErr = nil
-		}
-	}
-
-	// getCredDefault is a function used by DefaultFunc below to
-	// get the default value for various parts of the credentials.
-	// This function properly handles loading the credentials, checking
-	// for errors, etc.
-	getCredDefault := func(def interface{}, f func() string) (interface{}, error) {
-		once.Do(getCreds)
-
-		// If there was an error, that is always first
-		if credErr != nil {
-			return nil, credErr
-		}
-
-		// If the value is empty string, return nil (not set)
-		val := f()
-		if val == "" {
-			return def, nil
-		}
-
-		return val, nil
-	}
-
 	// The actual provider
 	return &schema.Provider{
 		Schema: map[string]*schema.Schema{
 			"access_key": &schema.Schema{
 				Type:        schema.TypeString,
-				Required: true,
+				Optional:    true,
-				DefaultFunc: func() (interface{}, error) {
+				Default:     "",
-					return getCredDefault(nil, func() string {
-						return credVal.AccessKeyID
-					})
-				},
 				Description: descriptions["access_key"],
 			},
 
 			"secret_key": &schema.Schema{
 				Type:        schema.TypeString,
-				Required: true,
+				Optional:    true,
-				DefaultFunc: func() (interface{}, error) {
+				Default:     "",
-					return getCredDefault(nil, func() string {
-						return credVal.SecretAccessKey
-					})
-				},
 				Description: descriptions["secret_key"],
 			},
 
 			"token": &schema.Schema{
 				Type:        schema.TypeString,
 				Optional:    true,
-				DefaultFunc: func() (interface{}, error) {
+				Default:     "",
-					return getCredDefault("", func() string {
-						return credVal.SessionToken
-					})
-				},
 				Description: descriptions["token"],
 			},