From f1b95788b90847e785db9c13ee0ba144fe2f041e Mon Sep 17 00:00:00 2001 From: Alisdair McDiarmid Date: Fri, 11 Dec 2020 13:09:25 -0500 Subject: [PATCH] command: Add tests for terraform validate -json Also uncomment and fix some tests which had been skipped for a couple of years. Those validate cases work now! Note that these test cases and the JSON output are not especially minimized, making them snapshot/golden tests. The output looks correct at time of writing, and we don't expect to change validate significantly any time soon, but if we do there will be some churn here. --- .../incorrectmodulename/output.json | 133 ++++++++++++++++++ .../interpolation/output.json | 43 ++++++ .../missing_defined_var/output.json | 6 + .../missing_quote/output.json | 25 ++++ .../validate-invalid/missing_var/output.json | 43 ++++++ .../multiple_modules/output.json | 43 ++++++ .../multiple_providers/output.json | 25 ++++ .../multiple_resources/output.json | 25 ++++ command/testdata/validate-invalid/output.json | 25 ++++ .../validate-invalid/outputs/output.json | 43 ++++++ command/testdata/validate-valid/output.json | 6 + command/validate_test.go | 83 +++++++++-- 12 files changed, 490 insertions(+), 10 deletions(-) create mode 100644 command/testdata/validate-invalid/incorrectmodulename/output.json create mode 100644 command/testdata/validate-invalid/interpolation/output.json create mode 100644 command/testdata/validate-invalid/missing_defined_var/output.json create mode 100644 command/testdata/validate-invalid/missing_quote/output.json create mode 100644 command/testdata/validate-invalid/missing_var/output.json create mode 100644 command/testdata/validate-invalid/multiple_modules/output.json create mode 100644 command/testdata/validate-invalid/multiple_providers/output.json create mode 100644 command/testdata/validate-invalid/multiple_resources/output.json create mode 100644 command/testdata/validate-invalid/output.json create mode 100644 command/testdata/validate-invalid/outputs/output.json create mode 100644 command/testdata/validate-valid/output.json diff --git a/command/testdata/validate-invalid/incorrectmodulename/output.json b/command/testdata/validate-invalid/incorrectmodulename/output.json new file mode 100644 index 000000000..6fb8b2cad --- /dev/null +++ b/command/testdata/validate-invalid/incorrectmodulename/output.json @@ -0,0 +1,133 @@ +{ + "valid": false, + "error_count": 6, + "warning_count": 1, + "diagnostics": [ + { + "severity": "error", + "summary": "Missing required argument", + "detail": "The argument \"source\" is required, but no definition was found.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 1, + "column": 23, + "byte": 22 + }, + "end": { + "line": 1, + "column": 23, + "byte": 22 + } + } + }, + { + "severity": "error", + "summary": "Invalid module instance name", + "detail": "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 1, + "column": 8, + "byte": 7 + }, + "end": { + "line": 1, + "column": 22, + "byte": 21 + } + } + }, + { + "severity": "warning", + "summary": "Interpolation-only expressions are deprecated", + "detail": "Terraform 0.11 and earlier required all non-constant expressions to be provided via interpolation syntax, but this pattern is now deprecated. To silence this warning, remove the \"${ sequence from the start and the }\" sequence from the end of this expression, leaving just the inner expression.\n\nTemplate interpolation syntax is still used to construct strings from expressions when the template includes multiple interpolation sequences or a mixture of literal strings and interpolations. This deprecation applies only to templates that consist entirely of a single interpolation sequence.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 5, + "column": 12, + "byte": 55 + }, + "end": { + "line": 5, + "column": 31, + "byte": 74 + } + } + }, + { + "severity": "error", + "summary": "Variables not allowed", + "detail": "Variables may not be used here.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 5, + "column": 15, + "byte": 58 + }, + "end": { + "line": 5, + "column": 18, + "byte": 61 + } + } + }, + { + "severity": "error", + "summary": "Unsuitable value type", + "detail": "Unsuitable value: value must be known", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 5, + "column": 12, + "byte": 55 + }, + "end": { + "line": 5, + "column": 31, + "byte": 74 + } + } + }, + { + "severity": "error", + "summary": "Module not installed", + "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 4, + "column": 1, + "byte": 27 + }, + "end": { + "line": 4, + "column": 15, + "byte": 41 + } + } + }, + { + "severity": "error", + "summary": "Module not installed", + "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + "range": { + "filename": "testdata/validate-invalid/incorrectmodulename/main.tf", + "start": { + "line": 1, + "column": 1, + "byte": 0 + }, + "end": { + "line": 1, + "column": 22, + "byte": 21 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/interpolation/output.json b/command/testdata/validate-invalid/interpolation/output.json new file mode 100644 index 000000000..01c7815e9 --- /dev/null +++ b/command/testdata/validate-invalid/interpolation/output.json @@ -0,0 +1,43 @@ +{ + "valid": false, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Variables not allowed", + "detail": "Variables may not be used here.", + "range": { + "filename": "testdata/validate-invalid/interpolation/main.tf", + "start": { + "line": 6, + "column": 16, + "byte": 122 + }, + "end": { + "line": 6, + "column": 19, + "byte": 125 + } + } + }, + { + "severity": "error", + "summary": "Invalid expression", + "detail": "A single static variable reference is required: only attribute access and indexing with constant keys. No calculations, function calls, template expressions, etc are allowed here.", + "range": { + "filename": "testdata/validate-invalid/interpolation/main.tf", + "start": { + "line": 10, + "column": 17, + "byte": 197 + }, + "end": { + "line": 10, + "column": 44, + "byte": 224 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/missing_defined_var/output.json b/command/testdata/validate-invalid/missing_defined_var/output.json new file mode 100644 index 000000000..ac2d30361 --- /dev/null +++ b/command/testdata/validate-invalid/missing_defined_var/output.json @@ -0,0 +1,6 @@ +{ + "valid": true, + "error_count": 0, + "warning_count": 0, + "diagnostics": [] +} diff --git a/command/testdata/validate-invalid/missing_quote/output.json b/command/testdata/validate-invalid/missing_quote/output.json new file mode 100644 index 000000000..0d1abbbbc --- /dev/null +++ b/command/testdata/validate-invalid/missing_quote/output.json @@ -0,0 +1,25 @@ +{ + "valid": false, + "error_count": 1, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Invalid reference", + "detail": "A reference to a resource type must be followed by at least one attribute access, specifying the resource name.", + "range": { + "filename": "testdata/validate-invalid/missing_quote/main.tf", + "start": { + "line": 6, + "column": 14, + "byte": 110 + }, + "end": { + "line": 6, + "column": 18, + "byte": 114 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/missing_var/output.json b/command/testdata/validate-invalid/missing_var/output.json new file mode 100644 index 000000000..677e5f634 --- /dev/null +++ b/command/testdata/validate-invalid/missing_var/output.json @@ -0,0 +1,43 @@ +{ + "valid": false, + "error_count": 1, + "warning_count": 1, + "diagnostics": [ + { + "severity": "warning", + "summary": "Interpolation-only expressions are deprecated", + "detail": "Terraform 0.11 and earlier required all non-constant expressions to be provided via interpolation syntax, but this pattern is now deprecated. To silence this warning, remove the \"${ sequence from the start and the }\" sequence from the end of this expression, leaving just the inner expression.\n\nTemplate interpolation syntax is still used to construct strings from expressions when the template includes multiple interpolation sequences or a mixture of literal strings and interpolations. This deprecation applies only to templates that consist entirely of a single interpolation sequence.", + "range": { + "filename": "testdata/validate-invalid/missing_var/main.tf", + "start": { + "line": 6, + "column": 21, + "byte": 117 + }, + "end": { + "line": 6, + "column": 41, + "byte": 137 + } + } + }, + { + "severity": "error", + "summary": "Reference to undeclared input variable", + "detail": "An input variable with the name \"description\" has not been declared. This variable can be declared with a variable \"description\" {} block.", + "range": { + "filename": "testdata/validate-invalid/missing_var/main.tf", + "start": { + "line": 6, + "column": 24, + "byte": 120 + }, + "end": { + "line": 6, + "column": 39, + "byte": 135 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/multiple_modules/output.json b/command/testdata/validate-invalid/multiple_modules/output.json new file mode 100644 index 000000000..b6ece4bf3 --- /dev/null +++ b/command/testdata/validate-invalid/multiple_modules/output.json @@ -0,0 +1,43 @@ +{ + "valid": false, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate module call", + "detail": "A module call named \"multi_module\" was already defined at testdata/validate-invalid/multiple_modules/main.tf:1,1-22. Module calls must have unique names within a module.", + "range": { + "filename": "testdata/validate-invalid/multiple_modules/main.tf", + "start": { + "line": 5, + "column": 1, + "byte": 46 + }, + "end": { + "line": 5, + "column": 22, + "byte": 67 + } + } + }, + { + "severity": "error", + "summary": "Module not installed", + "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.", + "range": { + "filename": "testdata/validate-invalid/multiple_modules/main.tf", + "start": { + "line": 5, + "column": 1, + "byte": 46 + }, + "end": { + "line": 5, + "column": 22, + "byte": 67 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/multiple_providers/output.json b/command/testdata/validate-invalid/multiple_providers/output.json new file mode 100644 index 000000000..836882790 --- /dev/null +++ b/command/testdata/validate-invalid/multiple_providers/output.json @@ -0,0 +1,25 @@ +{ + "valid": false, + "error_count": 1, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate provider configuration", + "detail": "A default (non-aliased) provider configuration for \"aws\" was already given at testdata/validate-invalid/multiple_providers/main.tf:1,1-15. If multiple configurations are required, set the \"alias\" argument for alternative configurations.", + "range": { + "filename": "testdata/validate-invalid/multiple_providers/main.tf", + "start": { + "line": 7, + "column": 1, + "byte": 85 + }, + "end": { + "line": 7, + "column": 15, + "byte": 99 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/multiple_resources/output.json b/command/testdata/validate-invalid/multiple_resources/output.json new file mode 100644 index 000000000..f1f0a8b52 --- /dev/null +++ b/command/testdata/validate-invalid/multiple_resources/output.json @@ -0,0 +1,25 @@ +{ + "valid": false, + "error_count": 1, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Duplicate resource \"aws_instance\" configuration", + "detail": "A aws_instance resource named \"web\" was already declared at testdata/validate-invalid/multiple_resources/main.tf:1,1-30. Resource names must be unique per type in each module.", + "range": { + "filename": "testdata/validate-invalid/multiple_resources/main.tf", + "start": { + "line": 4, + "column": 1, + "byte": 35 + }, + "end": { + "line": 4, + "column": 30, + "byte": 64 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/output.json b/command/testdata/validate-invalid/output.json new file mode 100644 index 000000000..3eebf7634 --- /dev/null +++ b/command/testdata/validate-invalid/output.json @@ -0,0 +1,25 @@ +{ + "valid": false, + "error_count": 1, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Unsupported block type", + "detail": "Blocks of type \"resorce\" are not expected here. Did you mean \"resource\"?", + "range": { + "filename": "testdata/validate-invalid/main.tf", + "start": { + "line": 1, + "column": 1, + "byte": 0 + }, + "end": { + "line": 1, + "column": 8, + "byte": 7 + } + } + } + ] +} diff --git a/command/testdata/validate-invalid/outputs/output.json b/command/testdata/validate-invalid/outputs/output.json new file mode 100644 index 000000000..454ad3cac --- /dev/null +++ b/command/testdata/validate-invalid/outputs/output.json @@ -0,0 +1,43 @@ +{ + "valid": false, + "error_count": 2, + "warning_count": 0, + "diagnostics": [ + { + "severity": "error", + "summary": "Missing required argument", + "detail": "The argument \"value\" is required, but no definition was found.", + "range": { + "filename": "testdata/validate-invalid/outputs/main.tf", + "start": { + "line": 1, + "column": 18, + "byte": 17 + }, + "end": { + "line": 1, + "column": 18, + "byte": 17 + } + } + }, + { + "severity": "error", + "summary": "Unsupported argument", + "detail": "An argument named \"values\" is not expected here. Did you mean \"value\"?", + "range": { + "filename": "testdata/validate-invalid/outputs/main.tf", + "start": { + "line": 2, + "column": 3, + "byte": 21 + }, + "end": { + "line": 2, + "column": 9, + "byte": 27 + } + } + } + ] +} diff --git a/command/testdata/validate-valid/output.json b/command/testdata/validate-valid/output.json new file mode 100644 index 000000000..ac2d30361 --- /dev/null +++ b/command/testdata/validate-valid/output.json @@ -0,0 +1,6 @@ +{ + "valid": true, + "error_count": 0, + "warning_count": 0, + "diagnostics": [] +} diff --git a/command/validate_test.go b/command/validate_test.go index cc5242f2d..114a1a861 100644 --- a/command/validate_test.go +++ b/command/validate_test.go @@ -1,10 +1,14 @@ package command import ( + "encoding/json" + "io/ioutil" "os" + "path" "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/mitchellh/cli" "github.com/zclconf/go-cty/cty" @@ -28,6 +32,7 @@ func setupTest(fixturepath string, args ...string) (*cli.MockUi, int) { Attributes: map[string]*configschema.Attribute{ "device_index": {Type: cty.String, Optional: true}, "description": {Type: cty.String, Optional: true}, + "name": {Type: cty.String, Optional: true}, }, }, }, @@ -83,29 +88,25 @@ func TestValidateFailingCommand(t *testing.T) { } func TestValidateFailingCommandMissingQuote(t *testing.T) { - // FIXME: Re-enable once we've updated core for new data structures - t.Skip("test temporarily disabled until deep validate supports new config structures") - ui, code := setupTest("validate-invalid/missing_quote") if code != 1 { t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String()) } - if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "IDENT test") { - t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String()) + wantError := "Error: Invalid reference" + if !strings.Contains(ui.ErrorWriter.String(), wantError) { + t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String()) } } func TestValidateFailingCommandMissingVariable(t *testing.T) { - // FIXME: Re-enable once we've updated core for new data structures - t.Skip("test temporarily disabled until deep validate supports new config structures") - ui, code := setupTest("validate-invalid/missing_var") if code != 1 { t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String()) } - if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "config: unknown variable referenced: 'description'; define it with a 'variable' block") { - t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String()) + wantError := "Error: Reference to undeclared input variable" + if !strings.Contains(ui.ErrorWriter.String(), wantError) { + t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String()) } } @@ -197,3 +198,65 @@ func TestMissingDefinedVar(t *testing.T) { t.Fatalf("Should have passed: %d\n\n%s", code, ui.ErrorWriter.String()) } } + +func TestValidate_json(t *testing.T) { + tests := []struct { + path string + valid bool + }{ + {"validate-valid", true}, + {"validate-invalid", false}, + {"validate-invalid/missing_quote", false}, + {"validate-invalid/missing_var", false}, + {"validate-invalid/multiple_providers", false}, + {"validate-invalid/multiple_modules", false}, + {"validate-invalid/multiple_resources", false}, + {"validate-invalid/outputs", false}, + {"validate-invalid/incorrectmodulename", false}, + {"validate-invalid/interpolation", false}, + {"validate-invalid/missing_defined_var", true}, + } + + for _, tc := range tests { + t.Run(tc.path, func(t *testing.T) { + var want, got map[string]interface{} + + wantFile, err := os.Open(path.Join(testFixturePath(tc.path), "output.json")) + if err != nil { + t.Fatalf("failed to open output file: %s", err) + } + defer wantFile.Close() + wantBytes, err := ioutil.ReadAll(wantFile) + if err != nil { + t.Fatalf("failed to read output file: %s", err) + } + err = json.Unmarshal([]byte(wantBytes), &want) + if err != nil { + t.Fatalf("failed to unmarshal expected JSON: %s", err) + } + + ui, code := setupTest(tc.path, "-json") + + gotString := ui.OutputWriter.String() + err = json.Unmarshal([]byte(gotString), &got) + if err != nil { + t.Fatalf("failed to unmarshal actual JSON: %s", err) + } + + if !cmp.Equal(got, want) { + t.Errorf("wrong output:\n %v\n", cmp.Diff(got, want)) + t.Errorf("raw output:\n%s\n", gotString) + } + + if tc.valid && code != 0 { + t.Errorf("wrong exit code: want 0, got %d", code) + } else if !tc.valid && code != 1 { + t.Errorf("wrong exit code: want 1, got %d", code) + } + + if errorOutput := ui.ErrorWriter.String(); errorOutput != "" { + t.Errorf("unexpected error output:\n%s", errorOutput) + } + }) + } +}