lang/blocktoattr: ExpandedVariables function

Because we handle FixUpBlockAttrs after dynamic block expansion, when
resolving variables we unfortunately need to consider the possibility of
both dynamic block expansion _and_ the block attrs fixup.

To accommodate this we have a variant of dynblock.VariablesHCLDec that
instead walks using the configschema.Block representation of the schema
and applies the same opportunistic schema rewrite used by FixUpBlockAttrs
at each body encountered during the walk.

lang/blocktoattr/fixup.go

@@ -28,21 +28,10 @@ func FixUpBlockAttrs(body hcl.Body, schema *configschema.Block) hcl.Body {
 		schema = &configschema.Block{}
 	}
 
-	// We'll do a quick sniff first to see if there's even anything ambiguous
-	// in this schema. (We still need to wrap it even if not, just in case we
-	// need to do fixup inside nested blocks.
-	ambiguousNames := make(map[string]struct{})
-	for name, attrS := range schema.Attributes {
-		aty := attrS.Type
-		if (aty.IsListType() || aty.IsSetType()) && aty.ElementType().IsObjectType() {
-			ambiguousNames[name] = struct{}{}
-		}
-	}
-
 	return &fixupBody{
 		original: body,
 		schema:   schema,
-		names:    ambiguousNames,
+		names:    ambiguousNames(schema),
 	}
 }
 
@@ -91,64 +80,7 @@ func (b *fixupBody) MissingItemRange() hcl.Range {
 // in the given schema, but some attribute schemas may instead be replaced by
 // block header schemas.
 func (b *fixupBody) effectiveSchema(given *hcl.BodySchema) *hcl.BodySchema {
-	ret := &hcl.BodySchema{}
+	return effectiveSchema(given, b.original, b.names, true)
-
-	appearsAsBlock := make(map[string]struct{})
-	{
-		// We'll construct some throwaway schemas here just to probe for
-		// whether each of our ambiguous names seems to be being used as
-		// an attribute or a block. We need to check both because in JSON
-		// syntax we rely on the schema to decide between attribute or block
-		// interpretation and so JSON will always answer yes to both of
-		// these questions and we want to prefer the attribute interpretation
-		// in that case.
-		var probeSchema hcl.BodySchema
-
-		for name := range b.names {
-			probeSchema = hcl.BodySchema{
-				Attributes: []hcl.AttributeSchema{
-					{
-						Name: name,
-					},
-				},
-			}
-			content, _, _ := b.original.PartialContent(&probeSchema)
-			if _, exists := content.Attributes[name]; exists {
-				// Can decode as an attribute, so we'll go with that.
-				continue
-			}
-			probeSchema = hcl.BodySchema{
-				Blocks: []hcl.BlockHeaderSchema{
-					{
-						Type: name,
-					},
-				},
-			}
-			content, _, _ = b.original.PartialContent(&probeSchema)
-			if len(content.Blocks) > 0 {
-				// No attribute present and at least one block present, so
-				// we'll need to rewrite this one as a block for a successful
-				// result.
-				appearsAsBlock[name] = struct{}{}
-			}
-		}
-	}
-
-	for _, attrS := range given.Attributes {
-		if _, exists := appearsAsBlock[attrS.Name]; exists {
-			ret.Blocks = append(ret.Blocks, hcl.BlockHeaderSchema{
-				Type: attrS.Name,
-			})
-		} else {
-			ret.Attributes = append(ret.Attributes, attrS)
-		}
-	}
-
-	// Anything that is specified as a block type in the input schema remains
-	// that way by just passing through verbatim.
-	ret.Blocks = append(ret.Blocks, given.Blocks...)
-
-	return ret
 }
 
 func (b *fixupBody) fixupContent(content *hcl.BodyContent) *hcl.BodyContent {
@@ -252,20 +184,3 @@ func (e *fixupBlocksExpr) Range() hcl.Range {
 func (e *fixupBlocksExpr) StartRange() hcl.Range {
 	return e.blocks[0].DefRange
 }
-
-// schemaForCtyType converts a cty object type into an approximately-equivalent
-// configschema.Block. If the given type is not an object type then this
-// function will panic.
-func schemaForCtyType(ty cty.Type) *configschema.Block {
-	atys := ty.AttributeTypes()
-	ret := &configschema.Block{
-		Attributes: make(map[string]*configschema.Attribute, len(atys)),
-	}
-	for name, aty := range atys {
-		ret.Attributes[name] = &configschema.Attribute{
-			Type:     aty,
-			Optional: true,
-		}
-	}
-	return ret
-}

lang/blocktoattr/schema.go

@@ -0,0 +1,114 @@
+package blocktoattr
+
+import (
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func ambiguousNames(schema *configschema.Block) map[string]struct{} {
+	ambiguousNames := make(map[string]struct{})
+	for name, attrS := range schema.Attributes {
+		aty := attrS.Type
+		if (aty.IsListType() || aty.IsSetType()) && aty.ElementType().IsObjectType() {
+			ambiguousNames[name] = struct{}{}
+		}
+	}
+	return ambiguousNames
+}
+
+func effectiveSchema(given *hcl.BodySchema, body hcl.Body, ambiguousNames map[string]struct{}, dynamicExpanded bool) *hcl.BodySchema {
+	ret := &hcl.BodySchema{}
+
+	appearsAsBlock := make(map[string]struct{})
+	{
+		// We'll construct some throwaway schemas here just to probe for
+		// whether each of our ambiguous names seems to be being used as
+		// an attribute or a block. We need to check both because in JSON
+		// syntax we rely on the schema to decide between attribute or block
+		// interpretation and so JSON will always answer yes to both of
+		// these questions and we want to prefer the attribute interpretation
+		// in that case.
+		var probeSchema hcl.BodySchema
+
+		for name := range ambiguousNames {
+			probeSchema = hcl.BodySchema{
+				Attributes: []hcl.AttributeSchema{
+					{
+						Name: name,
+					},
+				},
+			}
+			content, _, _ := body.PartialContent(&probeSchema)
+			if _, exists := content.Attributes[name]; exists {
+				// Can decode as an attribute, so we'll go with that.
+				continue
+			}
+			probeSchema = hcl.BodySchema{
+				Blocks: []hcl.BlockHeaderSchema{
+					{
+						Type: name,
+					},
+				},
+			}
+			content, _, _ = body.PartialContent(&probeSchema)
+			if len(content.Blocks) > 0 {
+				// No attribute present and at least one block present, so
+				// we'll need to rewrite this one as a block for a successful
+				// result.
+				appearsAsBlock[name] = struct{}{}
+			}
+		}
+		if !dynamicExpanded {
+			// If we're deciding for a context where dynamic blocks haven't
+			// been expanded yet then we need to probe for those too.
+			probeSchema = hcl.BodySchema{
+				Blocks: []hcl.BlockHeaderSchema{
+					{
+						Type:       "dynamic",
+						LabelNames: []string{"type"},
+					},
+				},
+			}
+			content, _, _ := body.PartialContent(&probeSchema)
+			for _, block := range content.Blocks {
+				if _, exists := ambiguousNames[block.Labels[0]]; exists {
+					appearsAsBlock[block.Labels[0]] = struct{}{}
+				}
+			}
+		}
+	}
+
+	for _, attrS := range given.Attributes {
+		if _, exists := appearsAsBlock[attrS.Name]; exists {
+			ret.Blocks = append(ret.Blocks, hcl.BlockHeaderSchema{
+				Type: attrS.Name,
+			})
+		} else {
+			ret.Attributes = append(ret.Attributes, attrS)
+		}
+	}
+
+	// Anything that is specified as a block type in the input schema remains
+	// that way by just passing through verbatim.
+	ret.Blocks = append(ret.Blocks, given.Blocks...)
+
+	return ret
+}
+
+// schemaForCtyType converts a cty object type into an approximately-equivalent
+// configschema.Block. If the given type is not an object type then this
+// function will panic.
+func schemaForCtyType(ty cty.Type) *configschema.Block {
+	atys := ty.AttributeTypes()
+	ret := &configschema.Block{
+		Attributes: make(map[string]*configschema.Attribute, len(atys)),
+	}
+	for name, aty := range atys {
+		ret.Attributes[name] = &configschema.Attribute{
+			Type:     aty,
+			Optional: true,
+		}
+	}
+	return ret
+}

lang/blocktoattr/variables.go

@@ -0,0 +1,43 @@
+package blocktoattr
+
+import (
+	"github.com/hashicorp/hcl2/ext/dynblock"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcldec"
+	"github.com/hashicorp/terraform/configs/configschema"
+)
+
+// ExpandedVariables finds all of the global variables referenced in the
+// given body with the given schema while taking into account the possibilities
+// both of "dynamic" blocks being expanded and the possibility of certain
+// attributes being written instead as nested blocks as allowed by the
+// FixUpBlockAttrs function.
+//
+// This function exists to allow variables to be analyzed prior to dynamic
+// block expansion while also dealing with the fact that dynamic block expansion
+// might in turn produce nested blocks that are subject to FixUpBlockAttrs.
+//
+// This is intended as a drop-in replacement for dynblock.VariablesHCLDec,
+// which is itself a drop-in replacement for hcldec.Variables.
+func ExpandedVariables(body hcl.Body, schema *configschema.Block) []hcl.Traversal {
+	rootNode := dynblock.WalkVariables(body)
+	return walkVariables(rootNode, body, schema)
+}
+
+func walkVariables(node dynblock.WalkVariablesNode, body hcl.Body, schema *configschema.Block) []hcl.Traversal {
+	givenRawSchema := hcldec.ImpliedSchema(schema.DecoderSpec())
+	ambiguousNames := ambiguousNames(schema)
+	effectiveRawSchema := effectiveSchema(givenRawSchema, body, ambiguousNames, false)
+	vars, children := node.Visit(effectiveRawSchema)
+
+	for _, child := range children {
+		if blockS, exists := schema.BlockTypes[child.BlockTypeName]; exists {
+			vars = append(vars, walkVariables(child.Node, child.Body(), &blockS.Block)...)
+		} else if attrS, exists := schema.Attributes[child.BlockTypeName]; exists {
+			synthSchema := schemaForCtyType(attrS.Type.ElementType())
+			vars = append(vars, walkVariables(child.Node, child.Body(), synthSchema)...)
+		}
+	}
+
+	return vars
+}

lang/blocktoattr/variables_test.go

@@ -0,0 +1,140 @@
+package blocktoattr
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcl/hclsyntax"
+	hcljson "github.com/hashicorp/hcl2/hcl/json"
+	"github.com/hashicorp/terraform/configs/configschema"
+	"github.com/zclconf/go-cty/cty"
+)
+
+func TestExpandedVariables(t *testing.T) {
+	fooSchema := &configschema.Block{
+		Attributes: map[string]*configschema.Attribute{
+			"foo": {
+				Type: cty.List(cty.Object(map[string]cty.Type{
+					"bar": cty.String,
+				})),
+				Optional: true,
+			},
+		},
+	}
+
+	tests := map[string]struct {
+		src    string
+		json   bool
+		schema *configschema.Block
+		want   []hcl.Traversal
+	}{
+		"empty": {
+			src:    ``,
+			schema: &configschema.Block{},
+			want:   nil,
+		},
+		"attribute syntax": {
+			src: `
+foo = [
+  {
+    bar = baz
+  },
+]
+`,
+			schema: fooSchema,
+			want: []hcl.Traversal{
+				{
+					hcl.TraverseRoot{
+						Name: "baz",
+						SrcRange: hcl.Range{
+							Filename: "test.tf",
+							Start:    hcl.Pos{Line: 4, Column: 11, Byte: 23},
+							End:      hcl.Pos{Line: 4, Column: 14, Byte: 26},
+						},
+					},
+				},
+			},
+		},
+		"block syntax": {
+			src: `
+foo {
+  bar = baz
+}
+`,
+			schema: fooSchema,
+			want: []hcl.Traversal{
+				{
+					hcl.TraverseRoot{
+						Name: "baz",
+						SrcRange: hcl.Range{
+							Filename: "test.tf",
+							Start:    hcl.Pos{Line: 3, Column: 9, Byte: 15},
+							End:      hcl.Pos{Line: 3, Column: 12, Byte: 18},
+						},
+					},
+				},
+			},
+		},
+		"dynamic block syntax": {
+			src: `
+dynamic "foo" {
+  for_each = beep
+  content {
+    bar = baz
+  }
+}
+`,
+			schema: fooSchema,
+			want: []hcl.Traversal{
+				{
+					hcl.TraverseRoot{
+						Name: "beep",
+						SrcRange: hcl.Range{
+							Filename: "test.tf",
+							Start:    hcl.Pos{Line: 3, Column: 14, Byte: 30},
+							End:      hcl.Pos{Line: 3, Column: 18, Byte: 34},
+						},
+					},
+				},
+				{
+					hcl.TraverseRoot{
+						Name: "baz",
+						SrcRange: hcl.Range{
+							Filename: "test.tf",
+							Start:    hcl.Pos{Line: 5, Column: 11, Byte: 57},
+							End:      hcl.Pos{Line: 5, Column: 14, Byte: 60},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			var f *hcl.File
+			var diags hcl.Diagnostics
+			if test.json {
+				f, diags = hcljson.Parse([]byte(test.src), "test.tf.json")
+			} else {
+				f, diags = hclsyntax.ParseConfig([]byte(test.src), "test.tf", hcl.Pos{Line: 1, Column: 1})
+			}
+			if diags.HasErrors() {
+				for _, diag := range diags {
+					t.Errorf("unexpected diagnostic: %s", diag)
+				}
+				t.FailNow()
+			}
+
+			got := ExpandedVariables(f.Body, test.schema)
+
+			co := cmpopts.IgnoreUnexported(hcl.TraverseRoot{})
+			if !cmp.Equal(got, test.want, co) {
+				t.Errorf("wrong result\n%s", cmp.Diff(test.want, got, co))
+			}
+		})
+	}
+
+}