config: allow HCL2 experiment opt-in (build-time flag to enable)

Use the new HCL2 config loader when the opt-in comment #terraform:hcl2 is
present in a .tf file.

For now this is disabled for "normal" builds and enabled only if
explicitly configured via a linker flag during build. This is because it's
not yet in a good state to be released: the HCL2 loader produces RawConfig
objects that the validator and interpolator can't yet deal with, and so
using HCL2 for anything non-trivial currently causes Terraform to crash
in real use.

config/import_tree.go

@@ -1,8 +1,10 @@
 package config
 
 import (
+	"bufio"
 	"fmt"
 	"io"
+	"os"
 )
 
 // configurable is an interface that must be implemented by any configuration
@@ -27,15 +29,52 @@ type importTree struct {
 // imports.
 type fileLoaderFunc func(path string) (configurable, []string, error)
 
+// Set this to a non-empty value at link time to enable the HCL2 experiment.
+// This is not currently enabled for release builds.
+//
+// For example:
+//    go install -ldflags="-X github.com/hashicorp/terraform/config.enableHCL2Experiment=true" github.com/hashicorp/terraform
+var enableHCL2Experiment = ""
+
 // loadTree takes a single file and loads the entire importTree for that
 // file. This function detects what kind of configuration file it is an
 // executes the proper fileLoaderFunc.
 func loadTree(root string) (*importTree, error) {
 	var f fileLoaderFunc
-	switch ext(root) {
+
-	case ".tf", ".tf.json":
+	// HCL2 experiment is currently activated at build time via the linker.
-		f = loadFileHcl
+	// See the comment on this variable for more information.
-	default:
+	if enableHCL2Experiment == "" {
+		// Main-line behavior: always use the original HCL parser
+		switch ext(root) {
+		case ".tf", ".tf.json":
+			f = loadFileHcl
+		default:
+		}
+	} else {
+		// Experimental behavior: use the HCL2 parser if the opt-in comment
+		// is present.
+		switch ext(root) {
+		case ".tf":
+			// We need to sniff the file for the opt-in comment line to decide
+			// if the file is participating in the HCL2 experiment.
+			cf, err := os.Open(root)
+			if err != nil {
+				return nil, err
+			}
+			sc := bufio.NewScanner(cf)
+			for sc.Scan() {
+				if sc.Text() == "#terraform:hcl2" {
+					f = globalHCL2Loader.loadFile
+				}
+			}
+			if f == nil {
+				f = loadFileHcl
+			}
+		case ".tf.json":
+			f = loadFileHcl
+		default:
+		}
 	}
 
 	if f == nil {

config/import_tree_test.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestImportTreeHCL2Experiment(t *testing.T) {
+	// Can only run this test if we're built with the experiment enabled.
+	// Enable this test by passing the following option to "go test":
+	//    -ldflags="-X github.com/hashicorp/terraform/config.enableHCL2Experiment=true"
+	// See the comment associated with this flag variable for more information.
+	if enableHCL2Experiment == "" {
+		t.Skip("HCL2 experiment is not enabled")
+	}
+
+	t.Run("HCL not opted in", func(t *testing.T) {
+		// .tf file without opt-in should use the old HCL parser
+		imp, err := loadTree("test-fixtures/hcl2-experiment-switch/not-opted-in.tf")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tree, err := imp.ConfigTree()
+		if err != nil {
+			t.Fatalf("unexpected error loading not-opted-in.tf: %s", err)
+		}
+
+		cfg := tree.Config
+		if got, want := len(cfg.Locals), 1; got != want {
+			t.Fatalf("wrong number of locals %#v; want %#v", got, want)
+		}
+		if cfg.Locals[0].RawConfig.Raw == nil {
+			// Having RawConfig.Raw indicates the old loader
+			t.Fatal("local has no RawConfig.Raw")
+		}
+	})
+
+	t.Run("HCL opted in", func(t *testing.T) {
+		// .tf file with opt-in should use the new HCL2 parser
+		imp, err := loadTree("test-fixtures/hcl2-experiment-switch/opted-in.tf")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tree, err := imp.ConfigTree()
+		if err != nil {
+			t.Fatalf("unexpected error loading opted-in.tf: %s", err)
+		}
+
+		cfg := tree.Config
+		if got, want := len(cfg.Locals), 1; got != want {
+			t.Fatalf("wrong number of locals %#v; want %#v", got, want)
+		}
+		if cfg.Locals[0].RawConfig.Body == nil {
+			// Having RawConfig.Body indicates the new loader
+			t.Fatal("local has no RawConfig.Body")
+		}
+	})
+
+	t.Run("JSON ineligible", func(t *testing.T) {
+		// .tf.json file should always use the old HCL parser
+		imp, err := loadTree("test-fixtures/hcl2-experiment-switch/not-eligible.tf.json")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		tree, err := imp.ConfigTree()
+		if err != nil {
+			t.Fatalf("unexpected error loading not-eligible.tf.json: %s", err)
+		}
+
+		cfg := tree.Config
+		if got, want := len(cfg.Locals), 1; got != want {
+			t.Fatalf("wrong number of locals %#v; want %#v", got, want)
+		}
+		if cfg.Locals[0].RawConfig.Raw == nil {
+			// Having RawConfig.Raw indicates the old loader
+			t.Fatal("local has no RawConfig.Raw")
+		}
+	})
+}

config/test-fixtures/hcl2-experiment-switch/not-eligible.tf.json

@@ -0,0 +1,5 @@
+{
+    "locals": {
+        "foo": "baz"
+    }
+}

config/test-fixtures/hcl2-experiment-switch/not-opted-in.tf

@@ -0,0 +1,7 @@
+
+# The use of an equals to assign "locals" is something that would be rejected
+# by the HCL2 parser (equals is reserved for attributes only) and so we can
+# use it to verify that the old HCL parser was used.
+locals {
+  foo = "bar"
+}

config/test-fixtures/hcl2-experiment-switch/opted-in.tf

@@ -0,0 +1,7 @@
+#terraform:hcl2
+
+locals {
+  # This direct expression is something that would be rejected by the old HCL
+  # parser, so we can use it as a marker that the HCL2 parser was used.
+  foo = 1 + 2
+}