command: Add tests for terraform validate -json

Also uncomment and fix some tests which had been skipped for a couple of
years. Those validate cases work now!

Note that these test cases and the JSON output are not especially
minimized, making them snapshot/golden tests. The output looks correct
at time of writing, and we don't expect to change validate significantly
any time soon, but if we do there will be some churn here.

command/testdata/validate-invalid/incorrectmodulename/output.json

@@ -0,0 +1,133 @@
+{
+  "valid": false,
+  "error_count": 6,
+  "warning_count": 1,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Missing required argument",
+      "detail": "The argument \"source\" is required, but no definition was found.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 1,
+          "column": 23,
+          "byte": 22
+        },
+        "end": {
+          "line": 1,
+          "column": 23,
+          "byte": 22
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Invalid module instance name",
+      "detail": "A name must start with a letter or underscore and may contain only letters, digits, underscores, and dashes.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 1,
+          "column": 8,
+          "byte": 7
+        },
+        "end": {
+          "line": 1,
+          "column": 22,
+          "byte": 21
+        }
+      }
+    },
+    {
+      "severity": "warning",
+      "summary": "Interpolation-only expressions are deprecated",
+      "detail": "Terraform 0.11 and earlier required all non-constant expressions to be provided via interpolation syntax, but this pattern is now deprecated. To silence this warning, remove the \"${ sequence from the start and the }\" sequence from the end of this expression, leaving just the inner expression.\n\nTemplate interpolation syntax is still used to construct strings from expressions when the template includes multiple interpolation sequences or a mixture of literal strings and interpolations. This deprecation applies only to templates that consist entirely of a single interpolation sequence.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 5,
+          "column": 12,
+          "byte": 55
+        },
+        "end": {
+          "line": 5,
+          "column": 31,
+          "byte": 74
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Variables not allowed",
+      "detail": "Variables may not be used here.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 5,
+          "column": 15,
+          "byte": 58
+        },
+        "end": {
+          "line": 5,
+          "column": 18,
+          "byte": 61
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Unsuitable value type",
+      "detail": "Unsuitable value: value must be known",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 5,
+          "column": 12,
+          "byte": 55
+        },
+        "end": {
+          "line": 5,
+          "column": 31,
+          "byte": 74
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Module not installed",
+      "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 4,
+          "column": 1,
+          "byte": 27
+        },
+        "end": {
+          "line": 4,
+          "column": 15,
+          "byte": 41
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Module not installed",
+      "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.",
+      "range": {
+        "filename": "testdata/validate-invalid/incorrectmodulename/main.tf",
+        "start": {
+          "line": 1,
+          "column": 1,
+          "byte": 0
+        },
+        "end": {
+          "line": 1,
+          "column": 22,
+          "byte": 21
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/interpolation/output.json

@@ -0,0 +1,43 @@
+{
+  "valid": false,
+  "error_count": 2,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Variables not allowed",
+      "detail": "Variables may not be used here.",
+      "range": {
+        "filename": "testdata/validate-invalid/interpolation/main.tf",
+        "start": {
+          "line": 6,
+          "column": 16,
+          "byte": 122
+        },
+        "end": {
+          "line": 6,
+          "column": 19,
+          "byte": 125
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Invalid expression",
+      "detail": "A single static variable reference is required: only attribute access and indexing with constant keys. No calculations, function calls, template expressions, etc are allowed here.",
+      "range": {
+        "filename": "testdata/validate-invalid/interpolation/main.tf",
+        "start": {
+          "line": 10,
+          "column": 17,
+          "byte": 197
+        },
+        "end": {
+          "line": 10,
+          "column": 44,
+          "byte": 224
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/missing_defined_var/output.json

@@ -0,0 +1,6 @@
+{
+  "valid": true,
+  "error_count": 0,
+  "warning_count": 0,
+  "diagnostics": []
+}

command/testdata/validate-invalid/missing_quote/output.json

@@ -0,0 +1,25 @@
+{
+  "valid": false,
+  "error_count": 1,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Invalid reference",
+      "detail": "A reference to a resource type must be followed by at least one attribute access, specifying the resource name.",
+      "range": {
+        "filename": "testdata/validate-invalid/missing_quote/main.tf",
+        "start": {
+          "line": 6,
+          "column": 14,
+          "byte": 110
+        },
+        "end": {
+          "line": 6,
+          "column": 18,
+          "byte": 114
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/missing_var/output.json

@@ -0,0 +1,43 @@
+{
+  "valid": false,
+  "error_count": 1,
+  "warning_count": 1,
+  "diagnostics": [
+    {
+      "severity": "warning",
+      "summary": "Interpolation-only expressions are deprecated",
+      "detail": "Terraform 0.11 and earlier required all non-constant expressions to be provided via interpolation syntax, but this pattern is now deprecated. To silence this warning, remove the \"${ sequence from the start and the }\" sequence from the end of this expression, leaving just the inner expression.\n\nTemplate interpolation syntax is still used to construct strings from expressions when the template includes multiple interpolation sequences or a mixture of literal strings and interpolations. This deprecation applies only to templates that consist entirely of a single interpolation sequence.",
+      "range": {
+        "filename": "testdata/validate-invalid/missing_var/main.tf",
+        "start": {
+          "line": 6,
+          "column": 21,
+          "byte": 117
+        },
+        "end": {
+          "line": 6,
+          "column": 41,
+          "byte": 137
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Reference to undeclared input variable",
+      "detail": "An input variable with the name \"description\" has not been declared. This variable can be declared with a variable \"description\" {} block.",
+      "range": {
+        "filename": "testdata/validate-invalid/missing_var/main.tf",
+        "start": {
+          "line": 6,
+          "column": 24,
+          "byte": 120
+        },
+        "end": {
+          "line": 6,
+          "column": 39,
+          "byte": 135
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/multiple_modules/output.json

@@ -0,0 +1,43 @@
+{
+  "valid": false,
+  "error_count": 2,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Duplicate module call",
+      "detail": "A module call named \"multi_module\" was already defined at testdata/validate-invalid/multiple_modules/main.tf:1,1-22. Module calls must have unique names within a module.",
+      "range": {
+        "filename": "testdata/validate-invalid/multiple_modules/main.tf",
+        "start": {
+          "line": 5,
+          "column": 1,
+          "byte": 46
+        },
+        "end": {
+          "line": 5,
+          "column": 22,
+          "byte": 67
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Module not installed",
+      "detail": "This module is not yet installed. Run \"terraform init\" to install all modules required by this configuration.",
+      "range": {
+        "filename": "testdata/validate-invalid/multiple_modules/main.tf",
+        "start": {
+          "line": 5,
+          "column": 1,
+          "byte": 46
+        },
+        "end": {
+          "line": 5,
+          "column": 22,
+          "byte": 67
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/multiple_providers/output.json

@@ -0,0 +1,25 @@
+{
+  "valid": false,
+  "error_count": 1,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Duplicate provider configuration",
+      "detail": "A default (non-aliased) provider configuration for \"aws\" was already given at testdata/validate-invalid/multiple_providers/main.tf:1,1-15. If multiple configurations are required, set the \"alias\" argument for alternative configurations.",
+      "range": {
+        "filename": "testdata/validate-invalid/multiple_providers/main.tf",
+        "start": {
+          "line": 7,
+          "column": 1,
+          "byte": 85
+        },
+        "end": {
+          "line": 7,
+          "column": 15,
+          "byte": 99
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/multiple_resources/output.json

@@ -0,0 +1,25 @@
+{
+  "valid": false,
+  "error_count": 1,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Duplicate resource \"aws_instance\" configuration",
+      "detail": "A aws_instance resource named \"web\" was already declared at testdata/validate-invalid/multiple_resources/main.tf:1,1-30. Resource names must be unique per type in each module.",
+      "range": {
+        "filename": "testdata/validate-invalid/multiple_resources/main.tf",
+        "start": {
+          "line": 4,
+          "column": 1,
+          "byte": 35
+        },
+        "end": {
+          "line": 4,
+          "column": 30,
+          "byte": 64
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/output.json

@@ -0,0 +1,25 @@
+{
+  "valid": false,
+  "error_count": 1,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Unsupported block type",
+      "detail": "Blocks of type \"resorce\" are not expected here. Did you mean \"resource\"?",
+      "range": {
+        "filename": "testdata/validate-invalid/main.tf",
+        "start": {
+          "line": 1,
+          "column": 1,
+          "byte": 0
+        },
+        "end": {
+          "line": 1,
+          "column": 8,
+          "byte": 7
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-invalid/outputs/output.json

@@ -0,0 +1,43 @@
+{
+  "valid": false,
+  "error_count": 2,
+  "warning_count": 0,
+  "diagnostics": [
+    {
+      "severity": "error",
+      "summary": "Missing required argument",
+      "detail": "The argument \"value\" is required, but no definition was found.",
+      "range": {
+        "filename": "testdata/validate-invalid/outputs/main.tf",
+        "start": {
+          "line": 1,
+          "column": 18,
+          "byte": 17
+        },
+        "end": {
+          "line": 1,
+          "column": 18,
+          "byte": 17
+        }
+      }
+    },
+    {
+      "severity": "error",
+      "summary": "Unsupported argument",
+      "detail": "An argument named \"values\" is not expected here. Did you mean \"value\"?",
+      "range": {
+        "filename": "testdata/validate-invalid/outputs/main.tf",
+        "start": {
+          "line": 2,
+          "column": 3,
+          "byte": 21
+        },
+        "end": {
+          "line": 2,
+          "column": 9,
+          "byte": 27
+        }
+      }
+    }
+  ]
+}

command/testdata/validate-valid/output.json

@@ -0,0 +1,6 @@
+{
+  "valid": true,
+  "error_count": 0,
+  "warning_count": 0,
+  "diagnostics": []
+}

command/validate_test.go

@@ -1,10 +1,14 @@
 package command
 
 import (
+	"encoding/json"
+	"io/ioutil"
 	"os"
+	"path"
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/mitchellh/cli"
 	"github.com/zclconf/go-cty/cty"
 
@@ -28,6 +32,7 @@ func setupTest(fixturepath string, args ...string) (*cli.MockUi, int) {
 							Attributes: map[string]*configschema.Attribute{
 								"device_index": {Type: cty.String, Optional: true},
 								"description":  {Type: cty.String, Optional: true},
+								"name":         {Type: cty.String, Optional: true},
 							},
 						},
 					},
@@ -83,29 +88,25 @@ func TestValidateFailingCommand(t *testing.T) {
 }
 
 func TestValidateFailingCommandMissingQuote(t *testing.T) {
-	// FIXME: Re-enable once we've updated core for new data structures
-	t.Skip("test temporarily disabled until deep validate supports new config structures")
-
 	ui, code := setupTest("validate-invalid/missing_quote")
 
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "IDENT test") {
+	wantError := "Error: Invalid reference"
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
 func TestValidateFailingCommandMissingVariable(t *testing.T) {
-	// FIXME: Re-enable once we've updated core for new data structures
-	t.Skip("test temporarily disabled until deep validate supports new config structures")
-
 	ui, code := setupTest("validate-invalid/missing_var")
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "config: unknown variable referenced: 'description'; define it with a 'variable' block") {
+	wantError := "Error: Reference to undeclared input variable"
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -197,3 +198,65 @@ func TestMissingDefinedVar(t *testing.T) {
 		t.Fatalf("Should have passed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
 }
+
+func TestValidate_json(t *testing.T) {
+	tests := []struct {
+		path  string
+		valid bool
+	}{
+		{"validate-valid", true},
+		{"validate-invalid", false},
+		{"validate-invalid/missing_quote", false},
+		{"validate-invalid/missing_var", false},
+		{"validate-invalid/multiple_providers", false},
+		{"validate-invalid/multiple_modules", false},
+		{"validate-invalid/multiple_resources", false},
+		{"validate-invalid/outputs", false},
+		{"validate-invalid/incorrectmodulename", false},
+		{"validate-invalid/interpolation", false},
+		{"validate-invalid/missing_defined_var", true},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.path, func(t *testing.T) {
+			var want, got map[string]interface{}
+
+			wantFile, err := os.Open(path.Join(testFixturePath(tc.path), "output.json"))
+			if err != nil {
+				t.Fatalf("failed to open output file: %s", err)
+			}
+			defer wantFile.Close()
+			wantBytes, err := ioutil.ReadAll(wantFile)
+			if err != nil {
+				t.Fatalf("failed to read output file: %s", err)
+			}
+			err = json.Unmarshal([]byte(wantBytes), &want)
+			if err != nil {
+				t.Fatalf("failed to unmarshal expected JSON: %s", err)
+			}
+
+			ui, code := setupTest(tc.path, "-json")
+
+			gotString := ui.OutputWriter.String()
+			err = json.Unmarshal([]byte(gotString), &got)
+			if err != nil {
+				t.Fatalf("failed to unmarshal actual JSON: %s", err)
+			}
+
+			if !cmp.Equal(got, want) {
+				t.Errorf("wrong output:\n %v\n", cmp.Diff(got, want))
+				t.Errorf("raw output:\n%s\n", gotString)
+			}
+
+			if tc.valid && code != 0 {
+				t.Errorf("wrong exit code: want 0, got %d", code)
+			} else if !tc.valid && code != 1 {
+				t.Errorf("wrong exit code: want 1, got %d", code)
+			}
+
+			if errorOutput := ui.ErrorWriter.String(); errorOutput != "" {
+				t.Errorf("unexpected error output:\n%s", errorOutput)
+			}
+		})
+	}
+}