lang/funcs: templatefile function

This function is similar to the template_file data source offered by the
template provider, but having it built in to the language makes it more
convenient to use, allowing templates to be rendered from files anywhere
an inline template would normally be allowed:

    user_data = templatefile("${path.module}/userdata.tmpl", {
      hostname = format("petserver%02d", count.index)
    })

Unlike the template_file data source, this function allows values of any
type in its variables map, passing them through verbatim to the template.
Its tighter integration with Terraform also allows it to return better
error messages with source location information from the template itself.

The template_file data source was originally created to work around the
fact that HIL didn't have any support for map values at the time, and
even once map support was added it wasn't very usable. With HCL2
expressions, there's little reason left to use a data source to render
a template; the only remaining reason left to use template_file is to
render a template that is constructed dynamically during the Terraform
run, which is a very rare need.

lang/funcs/filesystem.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"unicode/utf8"
 
+	"github.com/hashicorp/hcl2/hcl"
+	"github.com/hashicorp/hcl2/hcl/hclsyntax"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
@@ -63,6 +65,122 @@ func MakeFileFunc(baseDir string, encBase64 bool) function.Function {
 	})
 }
 
+// MakeTemplateFileFunc constructs a function that takes a file path and
+// an arbitrary object of named values and attempts to render the referenced
+// file as a template using HCL template syntax.
+//
+// The template itself may recursively call other functions so a callback
+// must be provided to get access to those functions. The template cannot,
+// however, access any variables defined in the scope: it is restricted only to
+// those variables provided in the second function argument, to ensure that all
+// dependencies on other graph nodes can be seen before executing this function.
+//
+// As a special exception, a referenced template file may not recursively call
+// the templatefile function, since that would risk the same file being
+// included into itself indefinitely.
+func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Function) function.Function {
+
+	params := []function.Parameter{
+		{
+			Name: "path",
+			Type: cty.String,
+		},
+		{
+			Name: "vars",
+			Type: cty.DynamicPseudoType,
+		},
+	}
+
+	loadTmpl := func(fn string) (hcl.Expression, error) {
+		// We re-use File here to ensure the same filename interpretation
+		// as it does, along with its other safety checks.
+		tmplVal, err := File(baseDir, cty.StringVal(fn))
+		if err != nil {
+			return nil, err
+		}
+
+		expr, diags := hclsyntax.ParseTemplate([]byte(tmplVal.AsString()), fn, hcl.Pos{Line: 1, Column: 1})
+		if diags.HasErrors() {
+			return nil, diags
+		}
+
+		return expr, nil
+	}
+
+	renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
+		if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
+			return cty.DynamicVal, function.NewArgErrorf(2, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
+		}
+
+		ctx := &hcl.EvalContext{
+			Variables: varsVal.AsValueMap(),
+		}
+
+		// We'll pre-check references in the template here so we can give a
+		// more specialized error message than HCL would by default, so it's
+		// clearer that this problem is coming from a templatefile call.
+		for _, traversal := range expr.Variables() {
+			root := traversal.RootName()
+			if _, ok := ctx.Variables[root]; !ok {
+				return cty.DynamicVal, function.NewArgErrorf(2, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
+			}
+		}
+
+		givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
+		funcs := make(map[string]function.Function, len(givenFuncs))
+		for name, fn := range givenFuncs {
+			if name == "templatefile" {
+				// We stub this one out to prevent recursive calls.
+				funcs[name] = function.New(&function.Spec{
+					Params: params,
+					Type: func(args []cty.Value) (cty.Type, error) {
+						return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
+					},
+				})
+				continue
+			}
+			funcs[name] = fn
+		}
+		ctx.Functions = funcs
+
+		val, diags := expr.Value(ctx)
+		if diags.HasErrors() {
+			return cty.DynamicVal, diags
+		}
+		return val, nil
+	}
+
+	return function.New(&function.Spec{
+		Params: params,
+		Type: func(args []cty.Value) (cty.Type, error) {
+			if !(args[0].IsKnown() && args[1].IsKnown()) {
+				return cty.DynamicPseudoType, nil
+			}
+
+			// We'll render our template now to see what result type it produces.
+			// A template consisting only of a single interpolation an potentially
+			// return any type.
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicPseudoType, err
+			}
+
+			// This is safe even if args[1] contains unknowns because the HCL
+			// template renderer itself knows how to short-circuit those.
+			val, err := renderTmpl(expr, args[1])
+			return val.Type(), err
+		},
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicVal, err
+			}
+			return renderTmpl(expr, args[1])
+		},
+	})
+
+}
+
 // MakeFileExistsFunc constructs a function that takes a path
 // and determines whether a file exists at that path
 func MakeFileExistsFunc(baseDir string) function.Function {

lang/funcs/filesystem_test.go

@@ -7,6 +7,7 @@ import (
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
 )
 
 func TestFile(t *testing.T) {
@@ -52,6 +53,128 @@ func TestFile(t *testing.T) {
 	}
 }
 
+func TestTemplateFile(t *testing.T) {
+	tests := []struct {
+		Path cty.Value
+		Vars cty.Value
+		Want cty.Value
+		Err  bool
+	}{
+		{
+			cty.StringVal("testdata/hello.txt"),
+			cty.EmptyObjectVal,
+			cty.StringVal("Hello World"),
+			false,
+		},
+		{
+			cty.StringVal("testdata/icon.png"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			true, // Not valid UTF-8
+		},
+		{
+			cty.StringVal("testdata/missing"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			true, // no file exists
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name": cty.StringVal("Jodie"),
+			}),
+			cty.StringVal("Hello, Jodie!"),
+			false,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"name": cty.StringVal("Jimbo"),
+			}),
+			cty.StringVal("Hello, Jimbo!"),
+			false,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			true, // "name" is missing from the vars map
+		},
+		{
+			cty.StringVal("testdata/func.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("The items are a, b, c"),
+			false,
+		},
+		{
+			cty.StringVal("testdata/recursive.tmpl"),
+			cty.MapValEmpty(cty.String),
+			cty.NilVal,
+			true, // recursive templatefile call not allowed
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("- a\n- b\n- c\n"),
+			false,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.True,
+			}),
+			cty.NilVal,
+			true, // iteration over non-iterable value
+		},
+		{
+			cty.StringVal("testdata/bare.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"val": cty.True,
+			}),
+			cty.True, // since this template contains only an interpolation, its true value shines through
+			false,
+		},
+	}
+
+	templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function {
+		return map[string]function.Function{
+			"join":         JoinFunc,
+			"templatefile": MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("TemplateFile(%#v, %#v)", test.Path, test.Vars), func(t *testing.T) {
+			got, err := templateFileFn.Call([]cty.Value{test.Path, test.Vars})
+
+			if test.Err {
+				if err == nil {
+					t.Fatal("succeeded; want error")
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}
+
 func TestFileExists(t *testing.T) {
 	tests := []struct {
 		Path cty.Value

lang/funcs/testdata/bare.tmpl

@@ -0,0 +1 @@
+${val}

lang/funcs/testdata/func.tmpl

@@ -0,0 +1 @@
+The items are ${join(", ", list)}

lang/funcs/testdata/hello.tmpl

@@ -0,0 +1 @@
+Hello, ${name}!

lang/funcs/testdata/list.tmpl

@@ -0,0 +1,3 @@
+%{ for x in list ~}
+- ${x}
+%{ endfor ~}

lang/funcs/testdata/recursive.tmpl

@@ -0,0 +1 @@
+${templatefile("recursive.tmpl", {})}

lang/functions.go

@@ -100,6 +100,12 @@ func (s *Scope) Functions() map[string]function.Function {
 			"zipmap":       funcs.ZipmapFunc,
 		}
 
+		s.funcs["templatefile"] = funcs.MakeTemplateFileFunc(s.BaseDir, func() map[string]function.Function {
+			// The templatefile function prevents recursive calls to itself
+			// by copying this map and overwriting the "templatefile" entry.
+			return s.funcs
+		})
+
 		if s.PureOnly {
 			// Force our few impure functions to return unknown so that we
 			// can defer evaluating them until a later pass.

website/docs/configuration/functions/file.html.md

@@ -21,12 +21,12 @@ this function will interpret the file contents as UTF-8 encoded text and
 return the resulting Unicode characters. If the file contains invalid UTF-8
 sequences then this function will produce an error.
 
-This function can be used only with functions that already exist as static
+This function can be used only with files that already exist on disk
-files on disk at the beginning of a Terraform run. Language functions do not
+at the beginning of a Terraform run. Functions do not participate in the
-participate in the dependency graph, so this function cannot be used with
+dependency graph, so this function cannot be used with files that are generated
-files that are generated dynamically during a Terraform operation. We do not
+dynamically during a Terraform operation. We do not recommend using dynamic
-recommend using of dynamic local files in Terraform configurations, but in rare
+local files in Terraform configurations, but in rare situations where this is
-situations where this is necessary you can use
+necessary you can use
 [the `local_file` data source](/docs/providers/local/d/file.html)
 to read files while respecting resource dependencies.
 
@@ -44,3 +44,5 @@ Hello World
   interpreting the contents as UTF-8 text.
 * [`fileexists`](./fileexists.html) determines whether a file exists
   at a given path.
+* [`templatefile`](./templatefile.html) renders uses a file from disk as a
+  template.

website/docs/configuration/functions/templatefile.html.md

@@ -0,0 +1,66 @@
+---
+layout: "functions"
+page_title: "templatefile - Functions - Configuration Language"
+sidebar_current: "docs-funcs-file-templatefile"
+description: |-
+  The templatefile function reads the file at the given path and renders its
+  content as a template.
+---
+
+# `templatefile` Function
+
+`templatefile` reads the file at the given path and renders its content
+as a template using a supplied set of template variables.
+
+```hcl
+templatefile(path, vars)
+```
+
+The template syntax is the same as for
+[string templates](../expressions.html#string-templates) in the main Terraform
+language, including interpolation sequences delimited with `${` ... `}`.
+This function just allows longer template sequences to be factored out
+into a separate file for readability.
+
+The "vars" argument must be a map. Within the template file, each of the keys
+in the map is available as a variable for interpolation. The template may
+also use any other function available in the Terraform language, except that
+recursive calls to `templatefile` are not permitted.
+
+Strings in the Terraform language are sequences of Unicode characters, so
+this function will interpret the file contents as UTF-8 encoded text and
+return the resulting Unicode characters. If the file contains invalid UTF-8
+sequences then this function will produce an error.
+
+This function can be used only with files that already exist on disk at the
+beginning of a Terraform run. Functions do not participate in the dependency
+graph, so this function cannot be used with files that are generated
+dynamically during a Terraform operation. We do not recommend using dynamic
+templates in Terraform configurations, but in rare situations where this is
+necessary you can use
+[the `template_file` data source](/docs/providers/template/d/file.html)
+to render templates while respecting resource dependencies.
+
+## Examples
+
+Given a template file `backends.tmpl` with the following content:
+
+```
+%{ for addr in ip_addrs ~}
+backend ${addr}:${port}
+%{ endfor ~}
+```
+
+The `templatefile` function renders the template:
+
+```
+> templatefile("${path.module}/backends.tmpl", { port = 8080, ip_addrs = ["10.0.0.1", "10.0.0.2"] })
+backend 10.0.0.1:8080
+backend 10.0.0.2:8080
+
+```
+
+## Related Functions
+
+* [`file`](./file.html) reads a file from disk and returns its literal contents
+  without any template interpretation.

website/layouts/functions.erb

@@ -257,6 +257,10 @@
                 <a href="/docs/configuration/functions/filebase64.html">filebase64</a>
               </li>
 
+              <li<%= sidebar_current("docs-funcs-file-templatefile") %>>
+                <a href="/docs/configuration/functions/templatefile.html">templatefile</a>
+              </li>
+
             </ul>
           </li>