command: beginnings of new config loader in "terraform validate"

As part of some light reorganization of our commands, this new
implementation no longer does validation of variables and will thus avoid
the need to spin up a fully-valid context. Instead, its focus is on
validating the configuration itself, regardless of any variables, state,
etc.

This change anticipates us later adding a -validate-only flag to
"terraform plan" which will then take over the related use-case of
checking if a particular execution of Terraform is valid, _including_ the
state, variables, etc.

Although leaving variables out of validate feels pretty arbitrary today
while all of the variable sources are local anyway, we have plans to
allow per-workspace variables to be stored in the backend in future and
at that point it will no longer be possible to fully validate variables
without accessing the backend. The "terraform plan" command explicitly
requires access to the backend, while "terraform validate" is now
explicitly for local-only validation of a single module.

In a future commit this will be extended to do basic type checking of
the configuration based on provider schemas, etc.

command/test-fixtures/validate-invalid/main.tf

@@ -1,8 +1,8 @@
-resource "test_instance" "foo" {
+resorce "test_instance" "foo" { # Intentional typo to test error reporting
     ami = "bar"
 
     network_interface {
       device_index = 0
-      description = "Main network interface ${var.this_is_an_error}"
+      description = "Main network interface"
     }
 }

command/test-fixtures/validate-invalid/multiple_modules/main.tf

@@ -1,5 +1,7 @@
 module "multi_module" {
+  source = "./foo"
 }
 
 module "multi_module" {
+  source = "./foo"
 }

command/validate.go

@@ -6,9 +6,6 @@ import (
 	"strings"
 
 	"github.com/hashicorp/terraform/tfdiags"
-
-	"github.com/hashicorp/terraform/config"
-	"github.com/hashicorp/terraform/terraform"
 )
 
 // ValidateCommand is a Command implementation that validates the terraform files
@@ -23,10 +20,8 @@ func (c *ValidateCommand) Run(args []string) int {
 	if err != nil {
 		return 1
 	}
-	var checkVars bool
 
 	cmdFlags := c.Meta.flagSet("validate")
-	cmdFlags.BoolVar(&checkVars, "check-variables", true, "check-variables")
 	cmdFlags.Usage = func() {
 		c.Ui.Error(c.Help())
 	}
@@ -54,7 +49,7 @@ func (c *ValidateCommand) Run(args []string) int {
 		return 1
 	}
 
-	rtnCode := c.validate(dir, checkVars)
+	rtnCode := c.validate(dir)
 
 	return rtnCode
 }
@@ -67,53 +62,52 @@ func (c *ValidateCommand) Help() string {
 	helpText := `
 Usage: terraform validate [options] [dir]
 
-  Validate the terraform files in a directory. Validation includes a
+  Validate the configuration files in a directory, referring only to the
-  basic check of syntax as well as checking that all variables declared
+  configuration and not accessing any remote services such as remote state,
-  in the configuration are specified in one of the possible ways:
+  provider APIs, etc.
 
-      -var foo=...
+  Validate runs checks that verify whether a configuration is
-      -var-file=foo.vars
+  internally-consistent, regardless of any provided variables or existing
-      TF_VAR_foo environment variable
+  state. It is thus primarily useful for general verification of reusable
-      terraform.tfvars
+  modules, including correctness of attribute names and value types.
-      default value
+
+  To verify configuration in the context of a particular run (a particular
+  target workspace, operation variables, etc), use the following command
+  instead:
+      terraform plan -validate-only
+
+  It is safe to run this command automatically, for example as a post-save
+  check in a text editor or as a test step for a re-usable module in a CI
+  system.
+
+  Validation requires an initialized working directory with any referenced
+  plugins and modules installed. To initialize a working directory for
+  validation without accessing any configured remote backend, use:
+      terraform init -backend=false
 
   If dir is not specified, then the current directory will be used.
 
 Options:
 
-  -check-variables=true If set to true (default), the command will check
-                        whether all required variables have been specified.
-
   -no-color    If specified, output won't contain any color.
-
-  -var 'foo=bar'        Set a variable in the Terraform configuration. This
-                        flag can be set multiple times.
-
-  -var-file=foo         Set variables in the Terraform configuration from
-                        a file. If "terraform.tfvars" is present, it will be
-                        automatically loaded if this flag is not specified.
 `
 	return strings.TrimSpace(helpText)
 }
 
-func (c *ValidateCommand) validate(dir string, checkVars bool) int {
+func (c *ValidateCommand) validate(dir string) int {
 	var diags tfdiags.Diagnostics
 
-	cfg, err := config.LoadDir(dir)
+	_, cfgDiags := c.loadConfig(dir)
-	if err != nil {
+	diags = diags.Append(cfgDiags)
-		diags = diags.Append(err)
-		c.showDiagnostics(err)
-		return 1
-	}
-
-	diags = diags.Append(cfg.Validate())
 
 	if diags.HasErrors() {
 		c.showDiagnostics(diags)
 		return 1
 	}
 
-	if checkVars {
+	// TODO: run a validation walk once terraform.NewContext is updated
+	// to support new-style configuration.
+	/* old implementation of validation....
 	mod, modDiags := c.Module(dir)
 	diags = diags.Append(modDiags)
 	if modDiags.HasErrors() {
@@ -132,12 +126,11 @@ func (c *ValidateCommand) validate(dir string, checkVars bool) int {
 	}
 
 	diags = diags.Append(tfCtx.Validate())
-	}
+	*/
 
 	c.showDiagnostics(diags)
 	if diags.HasErrors() {
 		return 1
 	}
-
 	return 0
 }

command/validate_test.go

@@ -26,7 +26,7 @@ func setupTest(fixturepath string, args ...string) (*cli.MockUi, int) {
 
 func TestValidateCommand(t *testing.T) {
 	if ui, code := setupTest("validate-valid"); code != 0 {
-		t.Fatalf("bad: %d\n\n%s", code, ui.ErrorWriter.String())
+		t.Fatalf("unexpected non-successful exit code %d\n\n%s", code, ui.ErrorWriter.String())
 	}
 }
 
@@ -59,6 +59,9 @@ func TestValidateFailingCommand(t *testing.T) {
 }
 
 func TestValidateFailingCommandMissingQuote(t *testing.T) {
+	// FIXME: Re-enable once we've updated core for new data structures
+	t.Skip("test temporarily disabled until deep validate supports new config structures")
+
 	ui, code := setupTest("validate-invalid/missing_quote")
 
 	if code != 1 {
@@ -70,6 +73,9 @@ func TestValidateFailingCommandMissingQuote(t *testing.T) {
 }
 
 func TestValidateFailingCommandMissingVariable(t *testing.T) {
+	// FIXME: Re-enable once we've updated core for new data structures
+	t.Skip("test temporarily disabled until deep validate supports new config structures")
+
 	ui, code := setupTest("validate-invalid/missing_var")
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
@@ -84,8 +90,9 @@ func TestSameProviderMutipleTimesShouldFail(t *testing.T) {
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "provider.aws: multiple configurations present; only one configuration is allowed per provider") {
+	wantError := "Error: Duplicate provider configuration"
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -94,8 +101,9 @@ func TestSameModuleMultipleTimesShouldFail(t *testing.T) {
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "module \"multi_module\": module repeated multiple times") {
+	wantError := "Error: Duplicate module call"
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -104,8 +112,9 @@ func TestSameResourceMultipleTimesShouldFail(t *testing.T) {
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "aws_instance.web: resource repeated multiple times") {
+	wantError := `Error: Duplicate resource "aws_instance" configuration`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -114,8 +123,13 @@ func TestOutputWithoutValueShouldFail(t *testing.T) {
 	if code != 1 {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
-	if !strings.HasSuffix(strings.TrimSpace(ui.ErrorWriter.String()), "output \"myvalue\": missing required 'value' argument") {
+	wantError := `The attribute "value" is required, but no definition was found.`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
+	}
+	wantError = `An attribute named "values" is not expected here. Did you mean "value"?`
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -125,11 +139,13 @@ func TestModuleWithIncorrectNameShouldFail(t *testing.T) {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
 
-	if !strings.Contains(ui.ErrorWriter.String(), "module name must be a letter or underscore followed by only letters, numbers, dashes, and underscores") {
+	wantError := `Error: Invalid module instance name`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
-	if !strings.Contains(ui.ErrorWriter.String(), "module source cannot contain interpolations") {
+	wantError = `Error: Variables not allowed`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
@@ -139,27 +155,20 @@ func TestWronglyUsedInterpolationShouldFail(t *testing.T) {
 		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}
 
-	if !strings.Contains(ui.ErrorWriter.String(), "depends on value cannot contain interpolations") {
+	wantError := `Error: Variables not allowed`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
-	if !strings.Contains(ui.ErrorWriter.String(), "variable \"vairable_with_interpolation\": default may not contain interpolations") {
+	wantError = `A static variable reference is required.`
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
+	if !strings.Contains(ui.ErrorWriter.String(), wantError) {
+		t.Fatalf("Missing error string %q\n\n'%s'", wantError, ui.ErrorWriter.String())
 	}
 }
 
 func TestMissingDefinedVar(t *testing.T) {
 	ui, code := setupTest("validate-invalid/missing_defined_var")
-	if code != 1 {
+	// This is allowed because validate tests only that variables are referenced
-		t.Fatalf("Should have failed: %d\n\n%s", code, ui.ErrorWriter.String())
+	// correctly, not that they all have defined values.
-	}
-
-	if !strings.Contains(ui.ErrorWriter.String(), "Required variable not set:") {
-		t.Fatalf("Should have failed: %d\n\n'%s'", code, ui.ErrorWriter.String())
-	}
-}
-
-func TestMissingDefinedVarConfigOnly(t *testing.T) {
-	ui, code := setupTest("validate-invalid/missing_defined_var", "-check-variables=false")
 	if code != 0 {
 		t.Fatalf("Should have passed: %d\n\n%s", code, ui.ErrorWriter.String())
 	}