lang/funcs: Add fileset function

Reference: https://github.com/hashicorp/terraform/issues/16697

Enumerates a set of regular file names from a given glob pattern. Implemented via the Go stdlib `path/filepath.Glob()` functionality. Notably, stdlib does not support `**` or `{}` extended patterns. See also: https://github.com/golang/go/issues/11862

To support the extended glob patterns, it will require adding a dependency on a third party library or adding our own matching code.

lang/funcs/filesystem.go

@@ -207,6 +207,60 @@ func MakeFileExistsFunc(baseDir string) function.Function {
 	})
 }
 
+// MakeFileSetFunc constructs a function that takes a glob pattern
+// and enumerates a file set from that pattern
+func MakeFileSetFunc(baseDir string) function.Function {
+	return function.New(&function.Spec{
+		Params: []function.Parameter{
+			{
+				Name: "pattern",
+				Type: cty.String,
+			},
+		},
+		Type: function.StaticReturnType(cty.Set(cty.String)),
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			pattern := args[0].AsString()
+			pattern, err := homedir.Expand(pattern)
+			if err != nil {
+				return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to expand ~: %s", err)
+			}
+
+			if !filepath.IsAbs(pattern) {
+				pattern = filepath.Join(baseDir, pattern)
+			}
+
+			// Ensure that the path is canonical for the host OS
+			pattern = filepath.Clean(pattern)
+
+			matches, err := filepath.Glob(pattern)
+			if err != nil {
+				return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to glob pattern (%s): %s", pattern, err)
+			}
+
+			var matchVals []cty.Value
+			for _, match := range matches {
+				fi, err := os.Stat(match)
+
+				if err != nil {
+					return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to stat (%s): %s", match, err)
+				}
+
+				if !fi.Mode().IsRegular() {
+					continue
+				}
+
+				matchVals = append(matchVals, cty.StringVal(match))
+			}
+
+			if len(matchVals) == 0 {
+				return cty.SetValEmpty(cty.String), nil
+			}
+
+			return cty.SetVal(matchVals), nil
+		},
+	})
+}
+
 // BasenameFunc constructs a function that takes a string containing a filesystem path
 // and removes all except the last portion from it.
 var BasenameFunc = function.New(&function.Spec{
@@ -316,6 +370,16 @@ func FileExists(baseDir string, path cty.Value) (cty.Value, error) {
 	return fn.Call([]cty.Value{path})
 }
 
+// FileSet enumerates a set of files given a glob pattern
+//
+// The underlying function implementation works relative to a particular base
+// directory, so this wrapper takes a base directory string and uses it to
+// construct the underlying function before calling it.
+func FileSet(baseDir string, pattern cty.Value) (cty.Value, error) {
+	fn := MakeFileSetFunc(baseDir)
+	return fn.Call([]cty.Value{pattern})
+}
+
 // FileBase64 reads the contents of the file at the given path.
 //
 // The bytes from the file are encoded as base64 before returning.

lang/funcs/filesystem_test.go

@@ -224,6 +224,120 @@ func TestFileExists(t *testing.T) {
 	}
 }
 
+func TestFileSet(t *testing.T) {
+	tests := []struct {
+		Pattern cty.Value
+		Want    cty.Value
+		Err     bool
+	}{
+		{
+			cty.StringVal("testdata/missing"),
+			cty.SetValEmpty(cty.String),
+			false,
+		},
+		{
+			cty.StringVal("testdata/missing*"),
+			cty.SetValEmpty(cty.String),
+			false,
+		},
+		{
+			cty.StringVal("*/missing"),
+			cty.SetValEmpty(cty.String),
+			false,
+		},
+		{
+			cty.StringVal("testdata"),
+			cty.SetValEmpty(cty.String),
+			false,
+		},
+		{
+			cty.StringVal("testdata*"),
+			cty.SetValEmpty(cty.String),
+			false,
+		},
+		{
+			cty.StringVal("testdata/*.txt"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("testdata/hello.txt"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("testdata/hello.???"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("testdata/hello*"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.tmpl"),
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("*/hello.txt"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("*/*.txt"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("*/hello*"),
+			cty.SetVal([]cty.Value{
+				cty.StringVal("testdata/hello.tmpl"),
+				cty.StringVal("testdata/hello.txt"),
+			}),
+			false,
+		},
+		{
+			cty.StringVal("["),
+			cty.SetValEmpty(cty.String),
+			true,
+		},
+		{
+			cty.StringVal("\\"),
+			cty.SetValEmpty(cty.String),
+			true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("FileSet(\".\", %#v)", test.Pattern), func(t *testing.T) {
+			got, err := FileSet(".", test.Pattern)
+
+			if test.Err {
+				if err == nil {
+					t.Fatal("succeeded; want error")
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}
+
 func TestFileBase64(t *testing.T) {
 	tests := []struct {
 		Path cty.Value

lang/functions.go

@@ -56,6 +56,7 @@ func (s *Scope) Functions() map[string]function.Function {
 			"chunklist":        funcs.ChunklistFunc,
 			"file":             funcs.MakeFileFunc(s.BaseDir, false),
 			"fileexists":       funcs.MakeFileExistsFunc(s.BaseDir),
+			"fileset":          funcs.MakeFileSetFunc(s.BaseDir),
 			"filebase64":       funcs.MakeFileFunc(s.BaseDir, true),
 			"filebase64sha256": funcs.MakeFileBase64Sha256Func(s.BaseDir),
 			"filebase64sha512": funcs.MakeFileBase64Sha512Func(s.BaseDir),

lang/functions_test.go

@@ -279,6 +279,16 @@ func TestFunctions(t *testing.T) {
 			},
 		},
 
+		"fileset": {
+			{
+				`fileset("hello.*")`,
+				cty.SetVal([]cty.Value{
+					cty.StringVal("testdata/functions-test/hello.tmpl"),
+					cty.StringVal("testdata/functions-test/hello.txt"),
+				}),
+			},
+		},
+
 		"filebase64": {
 			{
 				`filebase64("hello.txt")`,

website/docs/configuration/functions/fileset.html.md

@@ -0,0 +1,48 @@
+---
+layout: "functions"
+page_title: "fileset - Functions - Configuration Language"
+sidebar_current: "docs-funcs-file-file-set"
+description: |-
+  The fileset function enumerates a set of regular file names given a pattern.
+---
+
+# `fileset` Function
+
+-> **Note:** This page is about Terraform 0.12 and later. For Terraform 0.11 and
+earlier, see
+[0.11 Configuration Language: Interpolation Syntax](../../configuration-0-11/interpolation.html).
+
+`fileset` enumerates a set of regular file names given a pattern.
+
+```hcl
+fileset(pattern)
+```
+
+Supported pattern matches:
+
+- `*` - matches any sequence of non-separator characters
+- `?` - matches any single non-separator character
+- `[RANGE]` - matches a range of characters
+- `[^RANGE]` - matches outside the range of characters
+
+Functions are evaluated during configuration parsing rather than at apply time,
+so this function can only be used with files that are already present on disk
+before Terraform takes any actions.
+
+## Examples
+
+```
+> fileset("${path.module}/*.txt")
+[
+  "path/to/module/hello.txt",
+  "path/to/module/world.txt",
+]
+```
+
+```hcl
+resource "example_thing" "example" {
+  for_each = fileset("${path.module}/files/*")
+
+  # other configuration using each.value
+}
+```

website/layouts/functions.erb

@@ -304,6 +304,10 @@
                 <a href="/docs/configuration/functions/fileexists.html">fileexists</a>
               </li>
 
+              <li>
+                <a href="/docs/configuration/functions/fileset.html">fileset</a>
+              </li>
+
               <li>
                 <a href="/docs/configuration/functions/filebase64.html">filebase64</a>
               </li>