492 lines
23 KiB
Go
492 lines
23 KiB
Go
package test
|
|
|
|
import (
|
|
"reflect"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
|
|
"github.com/hashicorp/terraform/helper/schema"
|
|
"github.com/hashicorp/terraform/terraform"
|
|
)
|
|
|
|
var dataprocClusterSchema = map[string]*schema.Schema{
|
|
"name": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"project": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"region": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Default: "global",
|
|
ForceNew: true,
|
|
},
|
|
|
|
"labels": {
|
|
Type: schema.TypeMap,
|
|
Optional: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
// GCP automatically adds two labels
|
|
// 'goog-dataproc-cluster-uuid'
|
|
// 'goog-dataproc-cluster-name'
|
|
Computed: true,
|
|
DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
|
|
if old != "" {
|
|
return true
|
|
}
|
|
return false
|
|
},
|
|
},
|
|
|
|
"tag_set": {
|
|
Type: schema.TypeSet,
|
|
Optional: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
Set: schema.HashString,
|
|
},
|
|
|
|
"cluster_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
|
|
"delete_autogen_bucket": {
|
|
Type: schema.TypeBool,
|
|
Optional: true,
|
|
Default: false,
|
|
Removed: "If you need a bucket that can be deleted, please create" +
|
|
"a new one and set the `staging_bucket` field",
|
|
},
|
|
|
|
"staging_bucket": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
},
|
|
"bucket": {
|
|
Type: schema.TypeString,
|
|
Computed: true,
|
|
},
|
|
|
|
"gce_cluster_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
|
|
"zone": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"network": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
ConflictsWith: []string{"cluster_config.0.gce_cluster_config.0.subnetwork"},
|
|
},
|
|
|
|
"subnetwork": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
ConflictsWith: []string{"cluster_config.0.gce_cluster_config.0.network"},
|
|
},
|
|
|
|
"tags": {
|
|
Type: schema.TypeSet,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
},
|
|
|
|
"service_account": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"service_account_scopes": {
|
|
Type: schema.TypeSet,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Schema{
|
|
Type: schema.TypeString,
|
|
},
|
|
},
|
|
|
|
"internal_ip_only": {
|
|
Type: schema.TypeBool,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Default: false,
|
|
},
|
|
|
|
"metadata": {
|
|
Type: schema.TypeMap,
|
|
Optional: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
ForceNew: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"master_config": &schema.Schema{
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"num_instances": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
},
|
|
|
|
"image_uri": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"machine_type": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"disk_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"num_local_ssds": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"boot_disk_size_gb": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"boot_disk_type": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Default: "pd-standard",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"accelerators": {
|
|
Type: schema.TypeSet,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"accelerator_type": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"accelerator_count": {
|
|
Type: schema.TypeInt,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"instance_names": {
|
|
Type: schema.TypeList,
|
|
Computed: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"preemptible_worker_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"num_instances": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
},
|
|
"disk_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"num_local_ssds": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"boot_disk_size_gb": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"boot_disk_type": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Default: "pd-standard",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"instance_names": {
|
|
Type: schema.TypeList,
|
|
Computed: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"software_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
MaxItems: 1,
|
|
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"image_version": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"override_properties": {
|
|
Type: schema.TypeMap,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
},
|
|
|
|
"properties": {
|
|
Type: schema.TypeMap,
|
|
Computed: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"initialization_action": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"script": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"timeout_sec": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Default: 300,
|
|
ForceNew: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"encryption_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"kms_key_name": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func TestDiffApply_dataprocCluster(t *testing.T) {
|
|
priorAttrs := map[string]string{
|
|
"cluster_config.#": "1",
|
|
"cluster_config.0.bucket": "dataproc-1dc18cb2-116e-4e92-85ea-ff63a1bf2745-us-central1",
|
|
"cluster_config.0.delete_autogen_bucket": "false",
|
|
"cluster_config.0.encryption_config.#": "0",
|
|
"cluster_config.0.gce_cluster_config.#": "1",
|
|
"cluster_config.0.gce_cluster_config.0.internal_ip_only": "false",
|
|
"cluster_config.0.gce_cluster_config.0.metadata.%": "0",
|
|
"cluster_config.0.gce_cluster_config.0.network": "https://www.googleapis.com/compute/v1/projects/hc-terraform-testing/global/networks/default",
|
|
"cluster_config.0.gce_cluster_config.0.service_account": "",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.#": "7",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1245378569": "https://www.googleapis.com/auth/bigtable.admin.table",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1328717722": "https://www.googleapis.com/auth/devstorage.read_write",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.1693978638": "https://www.googleapis.com/auth/devstorage.full_control",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.172152165": "https://www.googleapis.com/auth/logging.write",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.2401844655": "https://www.googleapis.com/auth/bigquery",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.299921284": "https://www.googleapis.com/auth/bigtable.data",
|
|
"cluster_config.0.gce_cluster_config.0.service_account_scopes.3804780973": "https://www.googleapis.com/auth/cloud.useraccounts.readonly",
|
|
"cluster_config.0.gce_cluster_config.0.subnetwork": "",
|
|
"cluster_config.0.gce_cluster_config.0.tags.#": "0",
|
|
"cluster_config.0.gce_cluster_config.0.zone": "us-central1-f",
|
|
"cluster_config.0.initialization_action.#": "0",
|
|
"cluster_config.0.master_config.#": "1",
|
|
"cluster_config.0.master_config.0.accelerators.#": "0",
|
|
"cluster_config.0.master_config.0.disk_config.#": "1",
|
|
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb": "500",
|
|
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type": "pd-standard",
|
|
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds": "0",
|
|
"cluster_config.0.master_config.0.image_uri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-1-3-deb9-20190228-000000-rc01",
|
|
"cluster_config.0.master_config.0.instance_names.#": "1",
|
|
"cluster_config.0.master_config.0.instance_names.0": "dproc-cluster-test-2ww3c60iww-m",
|
|
"cluster_config.0.master_config.0.machine_type": "n1-standard-4",
|
|
"cluster_config.0.master_config.0.num_instances": "1",
|
|
"cluster_config.0.preemptible_worker_config.#": "1",
|
|
"cluster_config.0.preemptible_worker_config.0.disk_config.#": "1",
|
|
"cluster_config.0.preemptible_worker_config.0.instance_names.#": "0",
|
|
"cluster_config.0.preemptible_worker_config.0.num_instances": "0",
|
|
"cluster_config.0.software_config.#": "1",
|
|
"cluster_config.0.software_config.0.image_version": "1.3.28-deb9",
|
|
"cluster_config.0.software_config.0.override_properties.%": "0",
|
|
"cluster_config.0.software_config.0.properties.%": "14",
|
|
"cluster_config.0.software_config.0.properties.capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair",
|
|
"cluster_config.0.software_config.0.properties.core:fs.gs.block.size": "134217728",
|
|
"cluster_config.0.software_config.0.properties.core:fs.gs.metadata.cache.enable": "false",
|
|
"cluster_config.0.software_config.0.properties.core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2",
|
|
"cluster_config.0.software_config.0.properties.distcp:mapreduce.map.java.opts": "-Xmx768m",
|
|
"cluster_config.0.software_config.0.properties.distcp:mapreduce.map.memory.mb": "1024",
|
|
"cluster_config.0.software_config.0.properties.distcp:mapreduce.reduce.java.opts": "-Xmx768m",
|
|
"cluster_config.0.software_config.0.properties.distcp:mapreduce.reduce.memory.mb": "1024",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.address": "0.0.0.0:9866",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.http.address": "0.0.0.0:9864",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.https.address": "0.0.0.0:9865",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.handler.count": "20",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.http-address": "0.0.0.0:9870",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.https-address": "0.0.0.0:9871",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.lifeline.rpc-address": "dproc-cluster-test-2ww3c60iww-m:8050",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.service.handler.count": "10",
|
|
"cluster_config.0.software_config.0.properties.hdfs:dfs.namenode.servicerpc-address": "dproc-cluster-test-2ww3c60iww-m:8051",
|
|
"cluster_config.0.software_config.0.properties.mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "3840",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.maps": "21",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.job.reduces": "7",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.cpu.vcores": "1",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.java.opts": "-Xmx2457m",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.map.memory.mb": "3072",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.cpu.vcores": "1",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.java.opts": "-Xmx2457m",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.reduce.memory.mb": "3072",
|
|
"cluster_config.0.software_config.0.properties.mapred:mapreduce.task.io.sort.mb": "256",
|
|
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2457m",
|
|
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1",
|
|
"cluster_config.0.software_config.0.properties.mapred:yarn.app.mapreduce.am.resource.mb": "3072",
|
|
"cluster_config.0.software_config.0.properties.presto-jvm:MaxHeapSize": "12288m",
|
|
"cluster_config.0.software_config.0.properties.presto:query.max-memory-per-node": "7372MB",
|
|
"cluster_config.0.software_config.0.properties.presto:query.max-total-memory-per-node": "7372MB",
|
|
"cluster_config.0.software_config.0.properties.spark-env:SPARK_DAEMON_MEMORY": "3840m",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.driver.maxResultSize": "1920m",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.driver.memory": "3840m",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.executor.cores": "2",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.executor.instances": "2",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.executor.memory": "5586m",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.scheduler.mode": "FAIR",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.sql.cbo.enabled": "true",
|
|
"cluster_config.0.software_config.0.properties.spark:spark.yarn.am.memory": "640m",
|
|
"cluster_config.0.software_config.0.properties.yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "3840",
|
|
"cluster_config.0.software_config.0.properties.yarn:yarn.nodemanager.resource.memory-mb": "12288",
|
|
"cluster_config.0.software_config.0.properties.yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400",
|
|
"cluster_config.0.software_config.0.properties.yarn:yarn.scheduler.maximum-allocation-mb": "12288",
|
|
"cluster_config.0.software_config.0.properties.yarn:yarn.scheduler.minimum-allocation-mb": "1024",
|
|
"cluster_config.0.staging_bucket": "",
|
|
"id": "dproc-cluster-test-ktbyrniu4e",
|
|
"labels.%": "4",
|
|
"labels.goog-dataproc-cluster-name": "dproc-cluster-test-ktbyrniu4e",
|
|
"labels.goog-dataproc-cluster-uuid": "d576c4e0-8fda-4ad1-abf5-ec951ab25855",
|
|
"labels.goog-dataproc-location": "us-central1",
|
|
"labels.key1": "value1",
|
|
"tag_set.#": "0",
|
|
}
|
|
|
|
diff := &terraform.InstanceDiff{
|
|
Attributes: map[string]*terraform.ResourceAttrDiff{
|
|
"labels.%": &terraform.ResourceAttrDiff{Old: "4", New: "1", NewComputed: false, NewRemoved: false, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
|
|
"labels.goog-dataproc-cluster-name": &terraform.ResourceAttrDiff{Old: "dproc-cluster-test-ktbyrniu4e", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
|
|
"labels.goog-dataproc-cluster-uuid": &terraform.ResourceAttrDiff{Old: "d576c4e0-8fda-4ad1-abf5-ec951ab25855", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
|
|
"labels.goog-dataproc-location": &terraform.ResourceAttrDiff{Old: "us-central1", New: "", NewComputed: false, NewRemoved: true, NewExtra: interface{}(nil), RequiresNew: false, Sensitive: false, Type: 0x0},
|
|
},
|
|
}
|
|
|
|
newAttrs, err := diff.Apply(priorAttrs, (&schema.Resource{Schema: dataprocClusterSchema}).CoreConfigSchema())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// the diff'ed labale elements should be removed
|
|
delete(priorAttrs, "labels.goog-dataproc-cluster-name")
|
|
delete(priorAttrs, "labels.goog-dataproc-cluster-uuid")
|
|
delete(priorAttrs, "labels.goog-dataproc-location")
|
|
priorAttrs["labels.%"] = "1"
|
|
|
|
// the missing required "name" should be added
|
|
priorAttrs["name"] = ""
|
|
|
|
if !reflect.DeepEqual(priorAttrs, newAttrs) {
|
|
t.Fatal(cmp.Diff(priorAttrs, newAttrs))
|
|
}
|
|
}
|