ResiLien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
terraform/builtin/providers/test/resource_dataproc_cluster_t...

492 lines
23 KiB
Go
Raw Normal View History

add test for complex schema diff apply This schema was taken from an actual google resource which often shows perpetual diffs.
2019-03-12 15:02:07 +01:00
package test
import (
"reflect"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/terraform"
)
var dataprocClusterSchema = map[string]*schema.Schema{
"name": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"project": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"region": {
Type: schema.TypeString,
Optional: true,
Default: "global",
ForceNew: true,
},
"labels": {
Type: schema.TypeMap,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
// GCP automatically adds two labels
// 'goog-dataproc-cluster-uuid'
// 'goog-dataproc-cluster-name'
Computed: true,
DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
if old != "" {
return true
}
return false
},
},
"tag_set": {
Type: schema.TypeSet,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
Set: schema.HashString,
},
"cluster_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"delete_autogen_bucket": {
Type: schema.TypeBool,
Optional: true,
Default: false,
Removed: "If you need a bucket that can be deleted, please create" +
"a new one and set the `staging_bucket` field",
},
"staging_bucket": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
},
"bucket": {
Type: schema.TypeString,
Computed: true,
},
"gce_cluster_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"zone": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"network": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
ConflictsWith: []string{"cluster_config.0.gce_cluster_config.0.subnetwork"},
},
"subnetwork": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
ConflictsWith: []string{"cluster_config.0.gce_cluster_config.0.network"},
},
"tags": {
Type: schema.TypeSet,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"service_account": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
},
"service_account_scopes": {
Type: schema.TypeSet,
Optional: true,
Computed: true,
ForceNew: true,
Elem: &schema.Schema{
Type: schema.TypeString,
},
},
"internal_ip_only": {
Type: schema.TypeBool,
Optional: true,
ForceNew: true,
Default: false,
},
"metadata": {
Type: schema.TypeMap,
Optional: true,
Elem: &schema.Schema{Type: schema.TypeString},
ForceNew: true,
},
},
},
},
"master_config": &schema.Schema{
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_instances": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
},
"image_uri": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"machine_type": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"disk_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_local_ssds": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
},
"boot_disk_size_gb": {
helper: remove a bunch of unused packages, types and functions With the SDK moving out into its own repository, a lot of the packages under "helper/" are no longer needed. We still need to keep around just enough legacy SDK to support the "test" provider and some little bits and bobs in the backends and provisioners, but a lot of this is now just dead code. One of the test provider tests was depending on some validation functions only because the schema in there was originally copied from a "real" provider. The validation rules are not actually important for the test, so I removed them. Otherwise, this removes only dead code.
2020-10-02 01:36:22 +02:00
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
add test for complex schema diff apply This schema was taken from an actual google resource which often shows perpetual diffs.
2019-03-12 15:02:07 +01:00
},
"boot_disk_type": {
helper: remove a bunch of unused packages, types and functions With the SDK moving out into its own repository, a lot of the packages under "helper/" are no longer needed. We still need to keep around just enough legacy SDK to support the "test" provider and some little bits and bobs in the backends and provisioners, but a lot of this is now just dead code. One of the test provider tests was depending on some validation functions only because the schema in there was originally copied from a "real" provider. The validation rules are not actually important for the test, so I removed them. Otherwise, this removes only dead code.
2020-10-02 01:36:22 +02:00
Type: schema.TypeString,
Optional: true,
ForceNew: true,
Default: "pd-standard",
add test for complex schema diff apply This schema was taken from an actual google resource which often shows perpetual diffs.
2019-03-12 15:02:07 +01:00
},
},
},
},
"accelerators": {
Type: schema.TypeSet,
Optional: true,
ForceNew: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"accelerator_type": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"accelerator_count": {
Type: schema.TypeInt,
Required: true,
ForceNew: true,
},
},
},
},
"instance_names": {
Type: schema.TypeList,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
},
},
},
"preemptible_worker_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_instances": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
},
"disk_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"num_local_ssds": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
},
"boot_disk_size_gb": {
helper: remove a bunch of unused packages, types and functions With the SDK moving out into its own repository, a lot of the packages under "helper/" are no longer needed. We still need to keep around just enough legacy SDK to support the "test" provider and some little bits and bobs in the backends and provisioners, but a lot of this is now just dead code. One of the test provider tests was depending on some validation functions only because the schema in there was originally copied from a "real" provider. The validation rules are not actually important for the test, so I removed them. Otherwise, this removes only dead code.
2020-10-02 01:36:22 +02:00
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
add test for complex schema diff apply This schema was taken from an actual google resource which often shows perpetual diffs.
2019-03-12 15:02:07 +01:00
},
"boot_disk_type": {
helper: remove a bunch of unused packages, types and functions With the SDK moving out into its own repository, a lot of the packages under "helper/" are no longer needed. We still need to keep around just enough legacy SDK to support the "test" provider and some little bits and bobs in the backends and provisioners, but a lot of this is now just dead code. One of the test provider tests was depending on some validation functions only because the schema in there was originally copied from a "real" provider. The validation rules are not actually important for the test, so I removed them. Otherwise, this removes only dead code.
2020-10-02 01:36:22 +02:00
Type: schema.TypeString,
Optional: true,
ForceNew: true,
Default: "pd-standard",
add test for complex schema diff apply This schema was taken from an actual google resource which often shows perpetual diffs.
2019-03-12 15:02:07 +01:00
},
},
},
},
"instance_names": {
Type: schema.TypeList,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
},
},
},
"software_config": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"image_version": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"override_properties": {
Type: schema.TypeMap,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"properties": {
Type: schema.TypeMap,
Computed: true,
},
},
},
},
"initialization_action": {
Type: schema.TypeList,
Optional: true,
ForceNew: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"script": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"timeout_sec": {
Type: schema.TypeInt,
Optional: true,
Default: 300,
ForceNew: true,
},
},
},
},
"encryption_config": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"kms_key_name": {
Type: schema.TypeString,
Required: true,
},
},
},
},
},
},
},
}
func TestDiffApply_dataprocCluster(t *testing.T) {
priorAttrs := map[string]string{
"cluster_config.#": "1",
"cluster_config.0.bucket": "dataproc-1dc18cb2-116e-4e92-85ea-ff63a1bf2745-us-central1",
"cluster_config.0.delete_autogen_bucket": "false",
"cluster_config.0.encryption_config.#": "0",
"cluster_config.0.gce_cluster_config.#": "1",
"cluster_config.0.gce_cluster_config.0.internal_ip_only": "false",
"cluster_config.0.gce_cluster_config.0.metadata.%": "0",
"cluster_config.0.gce_cluster_config.0.network": "https://www.googleapis.com/compute/v1/projects/hc-terraform-testing/global/networks/default",
"cluster_config.0.gce_cluster_config.0.service_account": "",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.#": "7",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1245378569": "https://www.googleapis.com/auth/bigtable.admin.table",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1328717722": "https://www.googleapis.com/auth/devstorage.read_write",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1693978638": "https://www.googleapis.com/auth/devstorage.full_control",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.172152165": "https://www.googleapis.com/auth/logging.write",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.2401844655": "https://www.googleapis.com/auth/bigquery",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.299921284": "https://www.googleapis.com/auth/bigtable.data",
"cluster_config.0.gce_cluster_config.0.service_account_scopes.3804780973": "https://www.googleapis.com/auth/cloud.useraccounts.readonly",
"cluster_config.0.gce_cluster_config.0.subnetwork": "",
"cluster_config.0.gce_cluster_config.0.tags.#": "0",
"cluster_config.0.gce_cluster_config.0.zone": "us-central1-f",
"cluster_config.0.initialization_action.#": "0",
"cluster_config.0.master_config.#": "1",
"cluster_config.0.master_config.0.accelerators.#": "0",
"cluster_config.0.master_config.0.disk_config.#": "1",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb": "500",
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type": "pd-standard",
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds": "0",
"cluster_config.0.master_config.0.image_uri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20190228-000000-rc01",
"cluster_config.0.master_config.0.instance_names.#": "1",
"cluster_config.0.master_config.0.instance_names.0": "dproc-cluster-test-2ww3c60iww-m",
"cluster_config.0.master_config.0.machine_type": "n1-standard-4",
"cluster_config.0.master_config.0.num_instances": "1",
"cluster_config.0.preemptible_worker_config.#": "1",
"cluster_config.0.preemptible_worker_config.0.disk_config.#": "1",
"cluster_config.0.preemptible_worker_config.0.instance_names.#": "0",
"cluster_config.0.preemptible_worker_config.0.num_instances": "0",
"cluster_config.0.software_config.#": "1",
"cluster_config.0.software_config.0.image_version": "1.3.28-deb9",
"cluster_config.0.software_config.0.override_properties.%": "0",
"cluster_config.0.software_config.0.properties.%": "14",
"cluster_config.0.software_config.0.properties.capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair",
"cluster_config.0.software_config.0.properties.core:fs.gs.block.size": "134217728",
"cluster_config.0.software_config.0.properties.core:fs.gs.metadata.cache.enable": "false",
"cluster_config.0.software_config.0.properties.core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2",
"cluster_config.0.software_config.0.properties.distcp:mapreduce.map.java.opts": "-Xmx768m",
"cluster_config.0.software_config.0.properties.distcp:mapreduce.map.memory.mb": "1024",
"cluster_config.0.software_config.0.properties.distcp:mapreduce.reduce.java.opts": "-Xmx768m",
"cluster_config.0.software_config.0.properties.distcp:mapreduce.reduce.memory.mb": "1024",
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.address": "0.0.0.0:9866",
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.http.address": "0.0.0.0:9864",
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.https.address": "0.0.0.0:9865",
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.handler.count": "20",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.http-address": "0.0.0.0:9870",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.https-address": "0.0.0.0:9871",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.lifeline.rpc-address": "dproc-cluster-test-2ww3c60iww-m:8050",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.service.handler.count": "10",
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.servicerpc-address": "dproc-cluster-test-2ww3c60iww-m:8051",
"cluster_config.0.software_config.0.properties.mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "3840",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.maps": "21",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.reduces": "7",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.cpu.vcores": "1",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.java.opts": "-Xmx2457m",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.memory.mb": "3072",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.cpu.vcores": "1",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.java.opts": "-Xmx2457m",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.memory.mb": "3072",
"cluster_config.0.software_config.0.properties.mapred:mapreduce.task.io.sort.mb": "256",
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2457m",
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1",
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.resource.mb": "3072",
"cluster_config.0.software_config.0.properties.presto-jvm:MaxHeapSize": "12288m",
"cluster_config.0.software_config.0.properties.presto:query.max-memory-per-node": "7372MB",
"cluster_config.0.software_config.0.properties.presto:query.max-total-memory-per-node": "7372MB",
"cluster_config.0.software_config.0.properties.spark-env:SPARK_DAEMON_MEMORY": "3840m",
"cluster_config.0.software_config.0.properties.spark:spark.driver.maxResultSize": "1920m",
"cluster_config.0.software_config.0.properties.spark:spark.driver.memory": "3840m",
"cluster_config.0.software_config.0.properties.spark:spark.executor.cores": "2",
"cluster_config.0.software_config.0.properties.spark:spark.executor.instances": "2",
"cluster_config.0.software_config.0.properties.spark:spark.executor.memory": "5586m",
"cluster_config.0.software_config.0.properties.spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1",
"cluster_config.0.software_config.0.properties.spark:spark.scheduler.mode": "FAIR",
"cluster_config.0.software_config.0.properties.spark:spark.sql.cbo.enabled": "true",
"cluster_config.0.software_config.0.properties.spark:spark.yarn.am.memory": "640m",
"cluster_config.0.software_config.0.properties.yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "3840",
"cluster_config.0.software_config.0.properties.yarn:yarn.nodemanager.resource.memory-mb": "12288",
"cluster_config.0.software_config.0.properties.yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400",
"cluster_config.0.software_config.0.properties.yarn:yarn.scheduler.maximum-allocation-mb": "12288",
"cluster_config.0.software_config.0.properties.yarn:yarn.scheduler.minimum-allocation-mb": "1024",
"cluster_config.0.staging_bucket": "",
"id": "dproc-cluster-test-ktbyrniu4e",
"labels.%": "4",
"labels.goog-dataproc-cluster-name": "dproc-cluster-test-ktbyrniu4e",
"labels.goog-dataproc-cluster-uuid": "d576c4e0-8fda-4ad1-abf5-ec951ab25855",
"labels.goog-dataproc-location": "us-central1",
"labels.key1": "value1",
"tag_set.#": "0",
}
diff := &terraform.InstanceDiff{
Attributes: map[string]*terraform.ResourceAttrDiff{
"labels.%": &terraform.ResourceAttrDiff{Old: "4", New: "1", NewComputed: false, NewRemoved: false, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
"labels.goog-dataproc-cluster-name": &terraform.ResourceAttrDiff{Old: "dproc-cluster-test-ktbyrniu4e", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
"labels.goog-dataproc-cluster-uuid": &terraform.ResourceAttrDiff{Old: "d576c4e0-8fda-4ad1-abf5-ec951ab25855", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
"labels.goog-dataproc-location": &terraform.ResourceAttrDiff{Old: "us-central1", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
},
}
newAttrs, err := diff.Apply(priorAttrs, (&schema.Resource{Schema: dataprocClusterSchema}).CoreConfigSchema())
if err != nil {
t.Fatal(err)
}
// the diff'ed labale elements should be removed
delete(priorAttrs, "labels.goog-dataproc-cluster-name")
delete(priorAttrs, "labels.goog-dataproc-cluster-uuid")
delete(priorAttrs, "labels.goog-dataproc-location")
priorAttrs["labels.%"] = "1"
// the missing required "name" should be added
priorAttrs["name"] = ""
if !reflect.DeepEqual(priorAttrs, newAttrs) {
t.Fatal(cmp.Diff(priorAttrs, newAttrs))
}
}