provider/aws: add additional custom endpoints

* Add option for custom CloudFormation, KMS, RDS, SNS, SQS endpoints
* Minor formatting to arrange internal endpoint variables alphabetically
* Minor refactoring for consistent internal endpoint variable naming
* Update CHANGELOG
* Update docs accordingly

CHANGELOG.md

@@ -4,6 +4,7 @@ BACKWARDS INCOMPATIBILITIES / NOTES:
 
 * provider/aws: Users of aws_cloudfront_distributions with custom_origins have been broken due to changes in the AWS API requiring   `OriginReadTimeout` being set for updates. This has been fixed and will show as a change in terraform plan / apply. [GH-13367]
 * provider/aws: Users of China and Gov clouds, cannot use the new tagging of volumes created as part of aws_instances [GH-14055]
+* provider/aws: Add custom endpoint options for CloudFormation, CloudWatch, KMS, RDS, SNS, SQS
 
 FEATURES:
 

builtin/providers/aws/config.go

@@ -89,12 +89,20 @@ type Config struct {
 	AllowedAccountIds   []interface{}
 	ForbiddenAccountIds []interface{}
 
+	CloudFormationEndpoint   string
+	CloudWatchEndpoint       string
+	CloudWatchEventsEndpoint string
+	CloudWatchLogsEndpoint   string
 	DynamoDBEndpoint         string
-	KinesisEndpoint  string
 	Ec2Endpoint              string
-	IamEndpoint      string
 	ElbEndpoint              string
+	IamEndpoint              string
+	KinesisEndpoint          string
+	KmsEndpoint              string
+	RdsEndpoint              string
 	S3Endpoint               string
+	SnsEndpoint              string
+	SqsEndpoint              string
 	Insecure                 bool
 
 	SkipCredsValidation     bool
@@ -264,12 +272,20 @@ func (c *Config) Client() (interface{}, error) {
 	usEast1Sess := sess.Copy(&aws.Config{Region: aws.String("us-east-1")})
 
 	// Some services have user-configurable endpoints
+	awsCfSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.CloudFormationEndpoint)})
+	awsCwSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.CloudWatchEndpoint)})
+	awsCweSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.CloudWatchEventsEndpoint)})
+	awsCwlSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.CloudWatchLogsEndpoint)})
+	awsDynamoSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.DynamoDBEndpoint)})
 	awsEc2Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.Ec2Endpoint)})
 	awsElbSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.ElbEndpoint)})
 	awsIamSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.IamEndpoint)})
+	awsKinesisSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KinesisEndpoint)})
+	awsKmsSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KmsEndpoint)})
+	awsRdsSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.RdsEndpoint)})
 	awsS3Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.S3Endpoint)})
-	dynamoSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.DynamoDBEndpoint)})
+	awsSnsSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.SnsEndpoint)})
-	kinesisSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KinesisEndpoint)})
+	awsSqsSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.SqsEndpoint)})
 
 	// These two services need to be set up early so we can check on AccountID
 	client.iamconn = iam.New(awsIamSess)
@@ -312,12 +328,12 @@ func (c *Config) Client() (interface{}, error) {
 	client.apigateway = apigateway.New(sess)
 	client.appautoscalingconn = applicationautoscaling.New(sess)
 	client.autoscalingconn = autoscaling.New(sess)
-	client.cfconn = cloudformation.New(sess)
+	client.cfconn = cloudformation.New(awsCfSess)
 	client.cloudfrontconn = cloudfront.New(sess)
 	client.cloudtrailconn = cloudtrail.New(sess)
-	client.cloudwatchconn = cloudwatch.New(sess)
+	client.cloudwatchconn = cloudwatch.New(awsCwSess)
-	client.cloudwatcheventsconn = cloudwatchevents.New(sess)
+	client.cloudwatcheventsconn = cloudwatchevents.New(awsCweSess)
-	client.cloudwatchlogsconn = cloudwatchlogs.New(sess)
+	client.cloudwatchlogsconn = cloudwatchlogs.New(awsCwlSess)
 	client.codecommitconn = codecommit.New(sess)
 	client.codebuildconn = codebuild.New(sess)
 	client.codedeployconn = codedeploy.New(sess)
@@ -326,7 +342,7 @@ func (c *Config) Client() (interface{}, error) {
 	client.dmsconn = databasemigrationservice.New(sess)
 	client.codepipelineconn = codepipeline.New(sess)
 	client.dsconn = directoryservice.New(sess)
-	client.dynamodbconn = dynamodb.New(dynamoSess)
+	client.dynamodbconn = dynamodb.New(awsDynamoSess)
 	client.ecrconn = ecr.New(sess)
 	client.ecsconn = ecs.New(sess)
 	client.efsconn = efs.New(sess)
@@ -340,20 +356,20 @@ func (c *Config) Client() (interface{}, error) {
 	client.firehoseconn = firehose.New(sess)
 	client.inspectorconn = inspector.New(sess)
 	client.glacierconn = glacier.New(sess)
-	client.kinesisconn = kinesis.New(kinesisSess)
+	client.kinesisconn = kinesis.New(awsKinesisSess)
-	client.kmsconn = kms.New(sess)
+	client.kmsconn = kms.New(awsKmsSess)
 	client.lambdaconn = lambda.New(sess)
 	client.lightsailconn = lightsail.New(usEast1Sess)
 	client.opsworksconn = opsworks.New(sess)
 	client.r53conn = route53.New(usEast1Sess)
-	client.rdsconn = rds.New(sess)
+	client.rdsconn = rds.New(awsRdsSess)
 	client.redshiftconn = redshift.New(sess)
 	client.simpledbconn = simpledb.New(sess)
 	client.s3conn = s3.New(awsS3Sess)
 	client.sesConn = ses.New(sess)
 	client.sfnconn = sfn.New(sess)
-	client.snsconn = sns.New(sess)
+	client.snsconn = sns.New(awsSnsSess)
-	client.sqsconn = sqs.New(sess)
+	client.sqsconn = sqs.New(awsSqsSess)
 	client.ssmconn = ssm.New(sess)
 	client.wafconn = waf.New(sess)
 

builtin/providers/aws/provider.go

@@ -484,20 +484,36 @@ func init() {
 			"being executed. If the API request still fails, an error is\n" +
 			"thrown.",
 
+		"cloudformation_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
+		"cloudwatch_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
+		"cloudwatchevents_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
+		"cloudwatchlogs_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
 		"dynamodb_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n" +
 			"It's typically used to connect to dynamodb-local.",
 
 		"kinesis_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n" +
 			"It's typically used to connect to kinesalite.",
 
+		"kms_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
 		"iam_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
 
 		"ec2_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
 
 		"elb_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
 
+		"rds_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
 		"s3_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
 
+		"sns_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
+		"sqs_endpoint": "Use this to override the default endpoint URL constructed from the `region`.\n",
+
 		"insecure": "Explicitly allow the provider to perform \"insecure\" SSL requests. If omitted," +
 			"default value is `false`",
 
@@ -574,12 +590,20 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) {
 
 	for _, endpointsSetI := range endpointsSet.List() {
 		endpoints := endpointsSetI.(map[string]interface{})
+		config.CloudFormationEndpoint = endpoints["cloudformation"].(string)
+		config.CloudWatchEndpoint = endpoints["cloudwatch"].(string)
+		config.CloudWatchEventsEndpoint = endpoints["cloudwatchevents"].(string)
+		config.CloudWatchLogsEndpoint = endpoints["cloudwatchlogs"].(string)
 		config.DynamoDBEndpoint = endpoints["dynamodb"].(string)
-		config.IamEndpoint = endpoints["iam"].(string)
 		config.Ec2Endpoint = endpoints["ec2"].(string)
 		config.ElbEndpoint = endpoints["elb"].(string)
+		config.IamEndpoint = endpoints["iam"].(string)
 		config.KinesisEndpoint = endpoints["kinesis"].(string)
+		config.KmsEndpoint = endpoints["kms"].(string)
+		config.RdsEndpoint = endpoints["rds"].(string)
 		config.S3Endpoint = endpoints["s3"].(string)
+		config.SnsEndpoint = endpoints["sns"].(string)
+		config.SqsEndpoint = endpoints["sqs"].(string)
 	}
 
 	if v, ok := d.GetOk("allowed_account_ids"); ok {
@@ -648,6 +672,30 @@ func endpointsSchema() *schema.Schema {
 		Optional: true,
 		Elem: &schema.Resource{
 			Schema: map[string]*schema.Schema{
+				"cloudwatch": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["cloudwatch_endpoint"],
+				},
+				"cloudwatchevents": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["cloudwatchevents_endpoint"],
+				},
+				"cloudwatchlogs": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["cloudwatchlogs_endpoint"],
+				},
+				"cloudformation": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["cloudformation_endpoint"],
+				},
 				"dynamodb": {
 					Type:        schema.TypeString,
 					Optional:    true,
@@ -680,12 +728,36 @@ func endpointsSchema() *schema.Schema {
 					Default:     "",
 					Description: descriptions["kinesis_endpoint"],
 				},
+				"kms": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["kms_endpoint"],
+				},
+				"rds": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["rds_endpoint"],
+				},
 				"s3": {
 					Type:        schema.TypeString,
 					Optional:    true,
 					Default:     "",
 					Description: descriptions["s3_endpoint"],
 				},
+				"sns": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["sns_endpoint"],
+				},
+				"sqs": {
+					Type:        schema.TypeString,
+					Optional:    true,
+					Default:     "",
+					Description: descriptions["sqs_endpoint"],
+				},
 			},
 		},
 		Set: endpointsToHash,
@@ -695,12 +767,20 @@ func endpointsSchema() *schema.Schema {
 func endpointsToHash(v interface{}) int {
 	var buf bytes.Buffer
 	m := v.(map[string]interface{})
+	buf.WriteString(fmt.Sprintf("%s-", m["cloudwatch"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["cloudwatchevents"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["cloudwatchlogs"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["cloudformation"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["dynamodb"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["iam"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["ec2"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["elb"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["kinesis"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["kms"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["rds"].(string)))
 	buf.WriteString(fmt.Sprintf("%s-", m["s3"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["sns"].(string)))
+	buf.WriteString(fmt.Sprintf("%s-", m["sqs"].(string)))
 
 	return hashcode.String(buf.String())
 }

website/source/docs/providers/aws/index.html.markdown

@@ -234,6 +234,22 @@ in excess of those allowed by the access policy of the role that is being assume
 
 Nested `endpoints` block supports the following:
 
+* `cloudwatch` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom CloudWatch endpoints.
+
+* `cloudwatchevents` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom CloudWatchEvents endpoints.
+
+* `cloudwatchlogs` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom CloudWatchLogs endpoints.
+
+* `cloudformation` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom CloudFormation endpoints.
+
 * `dynamodb` - (Optional) Use this to override the default endpoint
   URL constructed from the `region`. It's typically used to connect to
   `dynamodb-local`.
@@ -242,6 +258,10 @@ Nested `endpoints` block supports the following:
   URL constructed from the `region`. It's typically used to connect to
   `kinesalite`.
 
+* `kms` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom KMS endpoints.
+
 * `iam` - (Optional) Use this to override the default endpoint
   URL constructed from the `region`. It's typically used to connect to
   custom IAM endpoints.
@@ -254,10 +274,22 @@ Nested `endpoints` block supports the following:
   URL constructed from the `region`. It's typically used to connect to
   custom ELB endpoints.
 
+* `rds` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom RDS endpoints.
+
 * `s3` - (Optional) Use this to override the default endpoint
   URL constructed from the `region`. It's typically used to connect to
   custom S3 endpoints.
 
+* `sns` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom SNS endpoints.
+
+* `sqs` - (Optional) Use this to override the default endpoint
+  URL constructed from the `region`. It's typically used to connect to
+  custom SQS endpoints.
+
 ## Getting the Account ID
 
 If you use either `allowed_account_ids` or `forbidden_account_ids`,