Add parallelism back into the tests

Running tests in parallel can help speed up overall test execution. Go
blocks parent tests while child tests run, so it does not fully fan out
as you might expect. It is noticably faster, though. Running 4 or more
concurrent processes knocks over a minute off the total execution time.
This commit is contained in:
Barrett Clark 2021-11-19 15:03:38 -06:00
parent 8b8fe2771f
commit c647b41d65
10 changed files with 112 additions and 87 deletions

View File

@ -13,6 +13,7 @@ import (
)
func Test_terraform_apply_autoApprove(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -180,20 +181,20 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
},
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -211,13 +212,13 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -233,7 +234,7 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -241,7 +242,7 @@ func Test_terraform_apply_autoApprove(t *testing.T) {
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -10,8 +10,8 @@ import (
)
func Test_backend_apply_before_init(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
// t.Parallel()
skipWithoutRemoteTerraformVersion(t)
cases := map[string]struct {
@ -72,20 +72,20 @@ func Test_backend_apply_before_init(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -103,13 +103,13 @@ func Test_backend_apply_before_init(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -125,7 +125,7 @@ func Test_backend_apply_before_init(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -133,7 +133,7 @@ func Test_backend_apply_before_init(t *testing.T) {
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -15,7 +15,7 @@ import (
)
const (
expectConsoleTimeout = 15 * time.Second
expectConsoleTimeout = 60 * time.Second * 3
)
type tfCommand struct {

View File

@ -10,8 +10,8 @@ import (
)
func Test_init_with_empty_tags(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
// t.Parallel()
skipWithoutRemoteTerraformVersion(t)
cases := map[string]struct {
@ -39,20 +39,20 @@ func Test_init_with_empty_tags(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -70,13 +70,13 @@ func Test_init_with_empty_tags(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -92,14 +92,14 @@ func Test_init_with_empty_tags(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
}
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -13,6 +13,7 @@ import (
)
func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -225,20 +226,20 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -256,13 +257,13 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -278,7 +279,7 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -286,7 +287,7 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -299,6 +300,7 @@ func Test_migrate_multi_to_tfc_cloud_name_strategy(t *testing.T) {
}
func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -513,20 +515,20 @@ func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -544,13 +546,13 @@ func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -569,7 +571,7 @@ func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
}
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -577,7 +579,7 @@ func Test_migrate_multi_to_tfc_cloud_tags_strategy(t *testing.T) {
err = cmd.Wait()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -12,6 +12,7 @@ import (
)
func Test_migrate_remote_backend_name_to_tfc_name(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -136,8 +137,10 @@ func Test_migrate_remote_backend_name_to_tfc_name(t *testing.T) {
}
func Test_migrate_remote_backend_name_to_tfc_same_name(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
ctx := context.Background()
operations := []operationSets{
{
@ -259,6 +262,7 @@ func Test_migrate_remote_backend_name_to_tfc_same_name(t *testing.T) {
}
func Test_migrate_remote_backend_name_to_tfc_name_different_org(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -388,6 +392,7 @@ func Test_migrate_remote_backend_name_to_tfc_name_different_org(t *testing.T) {
}
func Test_migrate_remote_backend_name_to_tfc_tags(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -522,6 +527,7 @@ func Test_migrate_remote_backend_name_to_tfc_tags(t *testing.T) {
}
func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_single_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -647,6 +653,7 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_single_workspace(t
}
func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_multi_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -801,6 +808,7 @@ func Test_migrate_remote_backend_prefix_to_tfc_name_strategy_multi_workspace(t *
}
func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_single_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -927,6 +935,7 @@ func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_single_workspace(t
}
func Test_migrate_remote_backend_prefix_to_tfc_tags_strategy_multi_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)

View File

@ -12,6 +12,7 @@ import (
)
func Test_migrate_single_to_tfc(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -129,20 +130,20 @@ func Test_migrate_single_to_tfc(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -160,13 +161,13 @@ func Test_migrate_single_to_tfc(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -182,7 +183,7 @@ func Test_migrate_single_to_tfc(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -190,7 +191,7 @@ func Test_migrate_single_to_tfc(t *testing.T) {
err = cmd.Wait()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -10,7 +10,9 @@ import (
)
func Test_migrate_tfc_to_other(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
cases := map[string]struct {
operations []operationSets
}{
@ -47,20 +49,20 @@ func Test_migrate_tfc_to_other(t *testing.T) {
}
for name, tc := range cases {
tc := tc
t.Run(name, func(t *testing.T) {
// t.Parallel()
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -78,13 +80,13 @@ func Test_migrate_tfc_to_other(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -100,14 +102,14 @@ func Test_migrate_tfc_to_other(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
}
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}
}

View File

@ -13,8 +13,10 @@ import (
)
func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
ctx := context.Background()
cases := map[string]struct {
@ -219,16 +221,18 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -248,13 +252,13 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -270,7 +274,7 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -278,7 +282,7 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err.Error())
subtest.Fatal(err.Error())
}
}
}
@ -291,6 +295,7 @@ func Test_migrate_tfc_to_tfc_single_workspace(t *testing.T) {
}
func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -464,16 +469,18 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -493,13 +500,13 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -515,7 +522,7 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
}
}
@ -523,7 +530,7 @@ func Test_migrate_tfc_to_tfc_multiple_workspace(t *testing.T) {
err = cmd.Wait()
if err != nil {
t.Fatal(err.Error())
subtest.Fatal(err.Error())
}
}
}

View File

@ -45,6 +45,7 @@ output "test_env" {
}
func Test_cloud_run_variables(t *testing.T) {
t.Parallel()
skipIfMissingEnvVar(t)
skipWithoutRemoteTerraformVersion(t)
@ -81,18 +82,20 @@ func Test_cloud_run_variables(t *testing.T) {
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
tc := tc // rebind tc into this lexical scope
t.Run(name, func(subtest *testing.T) {
subtest.Parallel()
organization, cleanup := createOrganization(t)
defer cleanup()
exp, err := expect.NewConsole(defaultOpts()...)
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer exp.Close()
tmpDir, err := ioutil.TempDir("", "terraform-test")
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
defer os.RemoveAll(tmpDir)
@ -112,13 +115,13 @@ func Test_cloud_run_variables(t *testing.T) {
err = cmd.Start()
if err != nil {
t.Fatal(err)
subtest.Fatal(err)
}
if tfCmd.expectedCmdOutput != "" {
got, err := exp.ExpectString(tfCmd.expectedCmdOutput)
if err != nil {
t.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
subtest.Fatalf("error while waiting for output\nwant: %s\nerror: %s\noutput\n%s", tfCmd.expectedCmdOutput, err, got)
}
}
@ -134,7 +137,7 @@ func Test_cloud_run_variables(t *testing.T) {
output := tfCmd.postInputOutput[i]
_, err := exp.ExpectString(output)
if err != nil {
t.Fatalf(`Expected command output "%s", but got %v `, tfCmd.expectedCmdOutput, err)
subtest.Fatalf(`Expected command output "%s", but got %v `, tfCmd.expectedCmdOutput, err)
}
}
}
@ -142,7 +145,7 @@ func Test_cloud_run_variables(t *testing.T) {
err = cmd.Wait()
if err != nil && !tfCmd.expectError {
t.Fatal(err)
subtest.Fatal(err)
}
}