config: Make HIL-based functions available to HCL2 via a shim

Terraform has a _lot_ of functions written against HIL's function API, and
we're not ready to rewrite them all yet, so instead we shim the HIL
function API to conform to the HCL2 (really: cty) function API and thus
allow most of our existing functions to work as expected when called from
HCL2-based config files.

Not all of the functions can be fully shimmed in this way due to depending
on HIL implementation details that we can't mimic through the HCL2 API.
We don't attempt to address that yet, and instead just let them fail when
called. We will eventually address this by using first-class HCL2
functions for these few cases, thus avoiding the HIL API altogether where
we need to. (The methodology for that is already illustrated here in the
provision of jsonencode and jsondecode functions that are HCL2-native.)

config/hcl2_shim_util.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/hashicorp/hil"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+
 	"github.com/hashicorp/hil/ast"
 
 	hcl2 "github.com/hashicorp/hcl2/hcl"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/convert"
 	"github.com/zclconf/go-cty/cty/function"
 )
 
@@ -127,26 +129,200 @@ func hcl2ValueFromConfigValue(v interface{}) cty.Value {
 	}
 }
 
+func hilVariableFromHCL2Value(v cty.Value) ast.Variable {
+	if v.IsNull() {
+		// Caller should guarantee/check this before calling
+		panic("Null values cannot be represented in HIL")
+	}
+	if !v.IsKnown() {
+		return ast.Variable{
+			Type:  ast.TypeUnknown,
+			Value: UnknownVariableValue,
+		}
+	}
+
+	switch v.Type() {
+	case cty.Bool:
+		return ast.Variable{
+			Type:  ast.TypeBool,
+			Value: v.True(),
+		}
+	case cty.Number:
+		v := configValueFromHCL2(v)
+		switch tv := v.(type) {
+		case int:
+			return ast.Variable{
+				Type:  ast.TypeInt,
+				Value: tv,
+			}
+		case float64:
+			return ast.Variable{
+				Type:  ast.TypeFloat,
+				Value: tv,
+			}
+		default:
+			// should never happen
+			panic("invalid return value for configValueFromHCL2")
+		}
+	case cty.String:
+		return ast.Variable{
+			Type:  ast.TypeString,
+			Value: v.AsString(),
+		}
+	}
+
+	if v.Type().IsListType() || v.Type().IsSetType() || v.Type().IsTupleType() {
+		l := make([]ast.Variable, 0, v.LengthInt())
+		it := v.ElementIterator()
+		for it.Next() {
+			_, ev := it.Element()
+			l = append(l, hilVariableFromHCL2Value(ev))
+		}
+		// If we were given a tuple then this could actually produce an invalid
+		// list with non-homogenous types, which we expect to be caught inside
+		// HIL just like a user-supplied non-homogenous list would be.
+		return ast.Variable{
+			Type:  ast.TypeList,
+			Value: l,
+		}
+	}
+
+	if v.Type().IsMapType() || v.Type().IsObjectType() {
+		l := make(map[string]ast.Variable)
+		it := v.ElementIterator()
+		for it.Next() {
+			ek, ev := it.Element()
+			l[ek.AsString()] = hilVariableFromHCL2Value(ev)
+		}
+		// If we were given an object then this could actually produce an invalid
+		// map with non-homogenous types, which we expect to be caught inside
+		// HIL just like a user-supplied non-homogenous map would be.
+		return ast.Variable{
+			Type:  ast.TypeMap,
+			Value: l,
+		}
+	}
+
+	// If we fall out here then we have some weird type that we haven't
+	// accounted for. This should never happen unless the caller is using
+	// capsule types, and we don't currently have any such types defined.
+	panic(fmt.Errorf("can't convert %#v to HIL variable", v))
+}
+
+func hcl2ValueFromHILVariable(v ast.Variable) cty.Value {
+	switch v.Type {
+	case ast.TypeList:
+		vals := make([]cty.Value, len(v.Value.([]ast.Variable)))
+		for i, ev := range v.Value.([]ast.Variable) {
+			vals[i] = hcl2ValueFromHILVariable(ev)
+		}
+		return cty.TupleVal(vals)
+	case ast.TypeMap:
+		vals := make(map[string]cty.Value, len(v.Value.(map[string]ast.Variable)))
+		for k, ev := range v.Value.(map[string]ast.Variable) {
+			vals[k] = hcl2ValueFromHILVariable(ev)
+		}
+		return cty.ObjectVal(vals)
+	default:
+		return hcl2ValueFromConfigValue(v.Value)
+	}
+}
+
+func hcl2TypeForHILType(hilType ast.Type) cty.Type {
+	switch hilType {
+	case ast.TypeAny:
+		return cty.DynamicPseudoType
+	case ast.TypeUnknown:
+		return cty.DynamicPseudoType
+	case ast.TypeBool:
+		return cty.Bool
+	case ast.TypeInt:
+		return cty.Number
+	case ast.TypeFloat:
+		return cty.Number
+	case ast.TypeString:
+		return cty.String
+	case ast.TypeList:
+		return cty.List(cty.DynamicPseudoType)
+	case ast.TypeMap:
+		return cty.Map(cty.DynamicPseudoType)
+	default:
+		return cty.NilType // equilvalent to ast.TypeInvalid
+	}
+}
+
 func hcl2InterpolationFuncs() map[string]function.Function {
 	hcl2Funcs := map[string]function.Function{}
 
 	for name, hilFunc := range Funcs() {
-		hcl2Funcs[name] = hcl2InterpolationFuncShim(&hilFunc)
+		hcl2Funcs[name] = hcl2InterpolationFuncShim(hilFunc)
 	}
 
+	// Some functions in the old world are dealt with inside langEvalConfig
+	// due to their legacy reliance on direct access to the symbol table.
+	// Since 0.7 they don't actually need it anymore and just ignore it,
+	// so we're cheating a bit here and exploiting that detail by passing nil.
+	hcl2Funcs["lookup"] = hcl2InterpolationFuncShim(interpolationFuncLookup(nil))
+	hcl2Funcs["keys"] = hcl2InterpolationFuncShim(interpolationFuncKeys(nil))
+	hcl2Funcs["values"] = hcl2InterpolationFuncShim(interpolationFuncValues(nil))
+
+	// As a bonus, we'll provide the JSON-handling functions from the cty
+	// function library since its "jsonencode" is more complete (doesn't force
+	// weird type conversions) and HIL's type system can't represent
+	// "jsondecode" at all. The result of jsondecode will eventually be forced
+	// to conform to the HIL type system on exit into the rest of Terraform due
+	// to our shimming right now, but it should be usable for decoding _within_
+	// an expression.
+	hcl2Funcs["jsonencode"] = stdlib.JSONEncodeFunc
+	hcl2Funcs["jsondecode"] = stdlib.JSONDecodeFunc
+
 	return hcl2Funcs
 }
 
-func hcl2InterpolationFuncShim(hilFunc *ast.Function) function.Function {
+func hcl2InterpolationFuncShim(hilFunc ast.Function) function.Function {
 	spec := &function.Spec{}
+
+	for i, hilArgType := range hilFunc.ArgTypes {
+		spec.Params = append(spec.Params, function.Parameter{
+			Type: hcl2TypeForHILType(hilArgType),
+			Name: fmt.Sprintf("arg%d", i+1), // HIL args don't have names, so we'll fudge it
+		})
+	}
+
+	if hilFunc.Variadic {
+		spec.VarParam = &function.Parameter{
+			Type: hcl2TypeForHILType(hilFunc.VariadicType),
+			Name: "varargs", // HIL args don't have names, so we'll fudge it
+		}
+	}
+
+	spec.Type = func(args []cty.Value) (cty.Type, error) {
+		return hcl2TypeForHILType(hilFunc.ReturnType), nil
+	}
 	spec.Impl = func(args []cty.Value, retType cty.Type) (cty.Value, error) {
 		hilArgs := make([]interface{}, len(args))
 		for i, arg := range args {
-			rv := configValueFromHCL2(arg)
+			hilV := hilVariableFromHCL2Value(arg)
-			hilV, err := hil.InterfaceToVariable(rv)
+
-			if err != nil {
+			// Although the cty function system does automatic type conversions
-				return cty.DynamicVal, err
+			// to match the argument types, cty doesn't distinguish int and
+			// float and so we may need to adjust here to ensure that the
+			// wrapped function gets exactly the Go type it was expecting.
+			var wantType ast.Type
+			if i < len(hilFunc.ArgTypes) {
+				wantType = hilFunc.ArgTypes[i]
+			} else {
+				wantType = hilFunc.VariadicType
 			}
+			switch {
+			case hilV.Type == ast.TypeInt && wantType == ast.TypeFloat:
+				hilV.Type = wantType
+				hilV.Value = float64(hilV.Value.(int))
+			case hilV.Type == ast.TypeFloat && wantType == ast.TypeInt:
+				hilV.Type = wantType
+				hilV.Value = int(hilV.Value.(float64))
+			}
+
 			// HIL functions actually expect to have the outermost variable
 			// "peeled" but any nested values (in lists or maps) will
 			// still have their ast.Variable wrapping.
@@ -154,20 +330,19 @@ func hcl2InterpolationFuncShim(hilFunc *ast.Function) function.Function {
 		}
 
 		hilResult, err := hilFunc.Callback(hilArgs)
-
-		// Just as on the way in, we get back a partially-peeled ast.Variable
-		// which we need to re-wrap in order to convert it back into what
-		// we're calling a "config value".
-
-		rr, err := hil.VariableToInterface(ast.Variable{
-			Type:  hilFunc.ReturnType,
-			Value: hilResult,
-		})
 		if err != nil {
 			return cty.DynamicVal, err
 		}
 
-		return hcl2ValueFromConfigValue(rr), nil
+		// Just as on the way in, we get back a partially-peeled ast.Variable
+		// which we need to re-wrap in order to convert it back into what
+		// we're calling a "config value".
+		rv := hcl2ValueFromHILVariable(ast.Variable{
+			Type:  hilFunc.ReturnType,
+			Value: hilResult,
+		})
+
+		return convert.Convert(rv, retType) // if result is unknown we'll force the correct type here
 	}
 	return function.New(spec)
 }

config/hcl2_shim_util_test.go

@@ -5,6 +5,8 @@ import (
 	"reflect"
 	"testing"
 
+	hcl2 "github.com/hashicorp/hcl2/hcl"
+	hcl2syntax "github.com/hashicorp/hcl2/hcl/hclsyntax"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -177,3 +179,170 @@ func TestHCL2ValueFromConfigValue(t *testing.T) {
 		})
 	}
 }
+
+func TestHCL2InterpolationFuncs(t *testing.T) {
+	// This is not a comprehensive test of all the functions (they are tested
+	// in interpolation_funcs_test.go already) but rather just calling a
+	// representative set via the HCL2 API to verify that the HCL2-to-HIL
+	// function shim is working as expected.
+	tests := []struct {
+		Expr string
+		Want cty.Value
+		Err  bool
+	}{
+		{
+			`upper("hello")`,
+			cty.StringVal("HELLO"),
+			false,
+		},
+		{
+			`abs(-2)`,
+			cty.NumberIntVal(2),
+			false,
+		},
+		{
+			`abs(-2.5)`,
+			cty.NumberFloatVal(2.5),
+			false,
+		},
+		{
+			`cidrsubnet("")`,
+			cty.DynamicVal,
+			true, // not enough arguments
+		},
+		{
+			`cidrsubnet("10.1.0.0/16", 8, 2)`,
+			cty.StringVal("10.1.2.0/24"),
+			false,
+		},
+		{
+			`concat([])`,
+			// Since HIL doesn't maintain element type information for list
+			// types, HCL2 can't either without elements to sniff.
+			cty.ListValEmpty(cty.DynamicPseudoType),
+			false,
+		},
+		{
+			`concat([], [])`,
+			cty.ListValEmpty(cty.DynamicPseudoType),
+			false,
+		},
+		{
+			`concat(["a"], ["b", "c"])`,
+			cty.ListVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+			}),
+			false,
+		},
+		{
+			`list()`,
+			cty.ListValEmpty(cty.DynamicPseudoType),
+			false,
+		},
+		{
+			`list("a", "b", "c")`,
+			cty.ListVal([]cty.Value{
+				cty.StringVal("a"),
+				cty.StringVal("b"),
+				cty.StringVal("c"),
+			}),
+			false,
+		},
+		{
+			`list(list("a"), list("b"), list("c"))`,
+			// The types emerge here in a bit of a strange tangle because of
+			// the guesswork we do when trying to recover lost information from
+			// HIL, but the rest of the language doesn't really care whether
+			// we use lists or tuples here as long as we are consistent with
+			// the type system invariants.
+			cty.ListVal([]cty.Value{
+				cty.TupleVal([]cty.Value{cty.StringVal("a")}),
+				cty.TupleVal([]cty.Value{cty.StringVal("b")}),
+				cty.TupleVal([]cty.Value{cty.StringVal("c")}),
+			}),
+			false,
+		},
+		{
+			`list(list("a"), "b")`,
+			cty.DynamicVal,
+			true, // inconsistent types
+		},
+		{
+			`length([])`,
+			cty.NumberIntVal(0),
+			false,
+		},
+		{
+			`length([2])`,
+			cty.NumberIntVal(1),
+			false,
+		},
+		{
+			`jsonencode(2)`,
+			cty.StringVal(`2`),
+			false,
+		},
+		{
+			`jsonencode(true)`,
+			cty.StringVal(`true`),
+			false,
+		},
+		{
+			`jsonencode("foo")`,
+			cty.StringVal(`"foo"`),
+			false,
+		},
+		{
+			`jsonencode({})`,
+			cty.StringVal(`{}`),
+			false,
+		},
+		{
+			`jsonencode([1])`,
+			cty.StringVal(`[1]`),
+			false,
+		},
+		{
+			`jsondecode("{}")`,
+			cty.EmptyObjectVal,
+			false,
+		},
+		{
+			`jsondecode("[5, true]")[0]`,
+			cty.NumberIntVal(5),
+			false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Expr, func(t *testing.T) {
+			expr, diags := hcl2syntax.ParseExpression([]byte(test.Expr), "", hcl2.Pos{Line: 1, Column: 1})
+			if len(diags) != 0 {
+				for _, diag := range diags {
+					t.Logf("- %s", diag)
+				}
+				t.Fatalf("unexpected diagnostics while parsing expression")
+			}
+
+			got, diags := expr.Value(&hcl2.EvalContext{
+				Functions: hcl2InterpolationFuncs(),
+			})
+			gotErr := diags.HasErrors()
+			if gotErr != test.Err {
+				if test.Err {
+					t.Errorf("expected errors but got none")
+				} else {
+					t.Errorf("unexpected errors")
+					for _, diag := range diags {
+						t.Logf("- %s", diag)
+					}
+				}
+			}
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\nexpr: %s\ngot:  %#v\nwant: %#v", test.Expr, got, test.Want)
+			}
+		})
+	}
+}