Add S3 endpoint override ability and expose S3 path style option (#7871)

* Overriding S3 endpoint - Enable specifying your own
  S3 api endpoint to override the default one, under
  endpoints.
* Force S3 path style - Expose this option from the aws-sdk-go
  configuration to the provider.

builtin/providers/aws/config.go

@@ -80,6 +80,7 @@ type Config struct {
 	SkipCredsValidation     bool
 	SkipRequestingAccountId bool
 	SkipMetadataApiCheck    bool
+	S3ForcePathStyle        bool
 }
 
 type AWSClient struct {
@@ -163,10 +164,11 @@ func (c *Config) Client() (interface{}, error) {
 		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)
 
 		awsConfig := &aws.Config{
-			Credentials: creds,
+			Credentials:      creds,
-			Region:      aws.String(c.Region),
+			Region:           aws.String(c.Region),
-			MaxRetries:  aws.Int(c.MaxRetries),
+			MaxRetries:       aws.Int(c.MaxRetries),
-			HTTPClient:  cleanhttp.DefaultClient(),
+			HTTPClient:       cleanhttp.DefaultClient(),
+			S3ForcePathStyle: aws.Bool(c.S3ForcePathStyle),
 		}
 
 		if logging.IsDebugOrHigher() {
@@ -199,6 +201,7 @@ func (c *Config) Client() (interface{}, error) {
 		awsEc2Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.Ec2Endpoint)})
 		awsElbSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.ElbEndpoint)})
 		awsIamSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.IamEndpoint)})
+		awsS3Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.S3Endpoint)})
 		dynamoSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.DynamoDBEndpoint)})
 		kinesisSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KinesisEndpoint)})
 
@@ -259,7 +262,7 @@ func (c *Config) Client() (interface{}, error) {
 		client.rdsconn = rds.New(sess)
 		client.redshiftconn = redshift.New(sess)
 		client.simpledbconn = simpledb.New(sess)
-		client.s3conn = s3.New(sess)
+		client.s3conn = s3.New(awsS3Sess)
 		client.sesConn = ses.New(sess)
 		client.snsconn = sns.New(sess)
 		client.sqsconn = sqs.New(sess)

builtin/providers/aws/provider.go

@@ -130,6 +130,13 @@ func Provider() terraform.ResourceProvider {
 				Default:     false,
 				Description: descriptions["skip_metadata_api_check"],
 			},
+
+			"s3_force_path_style": &schema.Schema{
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Default:     false,
+				Description: descriptions["s3_force_path_style"],
+			},
 		},
 
 		DataSourcesMap: map[string]*schema.Resource{
@@ -353,6 +360,8 @@ func init() {
 
 		"elb_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
 
+		"s3_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
 		"insecure": "Explicitly allow the provider to perform \"insecure\" SSL requests. If omitted," +
 			"default value is `false`",
 
@@ -364,6 +373,11 @@ func init() {
 
 		"skip_medatadata_api_check": "Skip the AWS Metadata API check. " +
 			"Used for AWS API implementations that do not have a metadata api endpoint.",
+
+		"s3_force_path_style": "Set this to true to force the request to use path-style addressing,\n" +
+			"i.e., http://s3.amazonaws.com/BUCKET/KEY. By default, the S3 client will\n" +
+			"use virtual hosted bucket addressing when possible\n" +
+			"(http://BUCKET.s3.amazonaws.com/KEY). Specific to the Amazon S3 service.",
 	}
 }
 
@@ -382,6 +396,7 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) {
 		SkipCredsValidation:     d.Get("skip_credentials_validation").(bool),
 		SkipRequestingAccountId: d.Get("skip_requesting_account_id").(bool),
 		SkipMetadataApiCheck:    d.Get("skip_metadata_api_check").(bool),
+		S3ForcePathStyle:        d.Get("s3_force_path_style").(bool),
 	}
 
 	endpointsSet := d.Get("endpoints").(*schema.Set)
@@ -391,6 +406,7 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) {
 		config.IamEndpoint = endpoints["iam"].(string)
 		config.Ec2Endpoint = endpoints["ec2"].(string)
 		config.ElbEndpoint = endpoints["elb"].(string)
+		config.S3Endpoint = endpoints["s3"].(string)
 	}
 
 	if v, ok := d.GetOk("allowed_account_ids"); ok {
@@ -433,6 +449,12 @@ func endpointsSchema() *schema.Schema {
 					Default:     "",
 					Description: descriptions["elb_endpoint"],
 				},
+				"s3": &schema.Schema{
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["s3_endpoint"],
+				},
 			},
 		},
 		Set: endpointsToHash,
@@ -445,6 +467,7 @@ func endpointsToHash(v interface{}) int {
 	buf.WriteString(fmt.Sprintf("%s-", m["iam"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["ec2"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["elb"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["s3"].(string)))
 
 	return hashcode.String(buf.String())
 }

website/source/docs/providers/aws/index.html.markdown

@@ -182,6 +182,10 @@ The following arguments are supported in the `provider` block:
   `true` prevents Terraform from authenticating via Metadata API - i.e. you may need to use other auth methods
   (static credentials set as ENV vars or config)
 
+* `s3_force_path_style` - (Optional) set this to true to force the request to use
+  path-style adressing, i.e., http://s3.amazonaws.com/BUCKET/KEY. By default, the
+  S3 client will use virtual hosted bucket addressing when possible
+  (http://BUCKET.s3.amazonaws.com/KEY). Specific to the Amazon S3 service.
 
 Nested `endpoints` block supports the followings:
 
@@ -197,6 +201,10 @@ Nested `endpoints` block supports the followings:
   URL constructed from the `region`. It's typically used to connect to
   custom elb endpoints.
 
+* `s3` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom s3 endpoints.
+
 ## Getting the Account ID
 
 If you use either `allowed_account_ids` or `forbidden_account_ids`,