From f721608e4ebcac09453ee646adf7545efc7abfec Mon Sep 17 00:00:00 2001 From: Quentin Machu Date: Thu, 13 Apr 2017 16:48:51 -0700 Subject: [PATCH 1/4] provider/template: Add a 'dir' resource to template entire directories When TerraForm is used to configure and deploy infrastructure applications that require dozens templated files, such as Kubernetes, it becomes extremely burdensome to template them individually: each of them requires a data source block as well as an upload/export (file provisioner, AWS S3, ...). Instead, this commit introduces a mean to template an entire folder of files (recursively), that can then be treated as a whole by any provider or provisioner that support directory inputs (such as the file provisioner, the archive provider, ...). This does not intend to make TerraForm a full-fledged templating system as the templating grammar and capabilities are left unchanged. This only aims at improving the user-experience of the existing templating provider by significantly reducing the overhead when several files are to be generated - without forcing the users to rely on external tools when these templates stay simple and that their generation in TerraForm is justified. --- builtin/providers/template/provider.go | 1 + .../template/resource_template_dir.go | 225 ++++++++++++++++++ .../template/resource_template_dir_test.go | 104 ++++++++ .../docs/providers/template/r/dir.html.md | 58 +++++ website/source/layouts/template.erb | 9 + 5 files changed, 397 insertions(+) create mode 100644 builtin/providers/template/resource_template_dir.go create mode 100644 builtin/providers/template/resource_template_dir_test.go create mode 100644 website/source/docs/providers/template/r/dir.html.md diff --git a/builtin/providers/template/provider.go b/builtin/providers/template/provider.go index ece6c9f34..fb340754d 100644 --- a/builtin/providers/template/provider.go +++ b/builtin/providers/template/provider.go @@ -20,6 +20,7 @@ func Provider() terraform.ResourceProvider { "template_cloudinit_config", dataSourceCloudinitConfig(), ), + "template_dir": resourceDir(), }, } } diff --git a/builtin/providers/template/resource_template_dir.go b/builtin/providers/template/resource_template_dir.go new file mode 100644 index 000000000..583926bb0 --- /dev/null +++ b/builtin/providers/template/resource_template_dir.go @@ -0,0 +1,225 @@ +package template + +import ( + "archive/tar" + "bytes" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/hashicorp/terraform/helper/pathorcontents" + "github.com/hashicorp/terraform/helper/schema" +) + +func resourceDir() *schema.Resource { + return &schema.Resource{ + Create: resourceTemplateDirCreate, + Read: resourceTemplateDirRead, + Delete: resourceTemplateDirDelete, + + Schema: map[string]*schema.Schema{ + "source_dir": { + Type: schema.TypeString, + Description: "Path to the directory where the files to template reside", + Required: true, + ForceNew: true, + }, + "vars": { + Type: schema.TypeMap, + Optional: true, + Default: make(map[string]interface{}), + Description: "Variables to substitute", + ValidateFunc: validateVarsAttribute, + ForceNew: true, + }, + "destination_dir": { + Type: schema.TypeString, + Description: "Path to the directory where the templated files will be written", + Required: true, + ForceNew: true, + }, + }, + } +} + +func resourceTemplateDirRead(d *schema.ResourceData, meta interface{}) error { + sourceDir := d.Get("source_dir").(string) + destinationDir := d.Get("destination_dir").(string) + + // If the output doesn't exist, mark the resource for creation. + if _, err := os.Stat(destinationDir); os.IsNotExist(err) { + d.SetId("") + return nil + } + + // If the combined hash of the input and output directories is different from + // the stored one, mark the resource for re-creation. + // + // The output directory is technically enough for the general case, but by + // hashing the input directory as well, we make development much easier: when + // a developer modifies one of the input files, the generation is + // re-triggered. + hash, err := generateID(sourceDir, destinationDir) + if err != nil { + return err + } + if hash != d.Id() { + d.SetId("") + return nil + } + + return nil +} + +func resourceTemplateDirCreate(d *schema.ResourceData, meta interface{}) error { + sourceDir := d.Get("source_dir").(string) + destinationDir := d.Get("destination_dir").(string) + vars := d.Get("vars").(map[string]interface{}) + + // Always delete the output first, otherwise files that got deleted from the + // input directory might still be present in the output afterwards. + if err := resourceTemplateDirDelete(d, meta); err != nil { + return err + } + + // Recursively crawl the input files/directories and generate the output ones. + err := filepath.Walk(sourceDir, func(p string, f os.FileInfo, err error) error { + if f.IsDir() { + return nil + } + if err != nil { + return err + } + + relPath, _ := filepath.Rel(sourceDir, p) + return generateDirFile(p, path.Join(destinationDir, relPath), f, vars) + }) + if err != nil { + return err + } + + // Compute ID. + hash, err := generateID(sourceDir, destinationDir) + if err != nil { + return err + } + d.SetId(hash) + + return nil +} + +func resourceTemplateDirDelete(d *schema.ResourceData, _ interface{}) error { + d.SetId("") + + destinationDir := d.Get("destination_dir").(string) + if _, err := os.Stat(destinationDir); os.IsNotExist(err) { + return nil + } + + if err := os.RemoveAll(destinationDir); err != nil { + return fmt.Errorf("could not delete directory %q: %s", destinationDir, err) + } + + return nil +} + +func generateDirFile(sourceDir, destinationDir string, f os.FileInfo, vars map[string]interface{}) error { + inputContent, _, err := pathorcontents.Read(sourceDir) + if err != nil { + return err + } + + outputContent, err := execute(inputContent, vars) + if err != nil { + return templateRenderError(fmt.Errorf("failed to render %v: %v", sourceDir, err)) + } + + outputDir := path.Dir(destinationDir) + if _, err := os.Stat(outputDir); err != nil { + if err := os.MkdirAll(outputDir, 0777); err != nil { + return err + } + } + + err = ioutil.WriteFile(destinationDir, []byte(outputContent), f.Mode()) + if err != nil { + return err + } + + return nil +} + +func generateID(sourceDir, destinationDir string) (string, error) { + inputHash, err := generateDirHash(sourceDir) + if err != nil { + return "", err + } + outputHash, err := generateDirHash(destinationDir) + if err != nil { + return "", err + } + checksum := sha1.Sum([]byte(inputHash + outputHash)) + return hex.EncodeToString(checksum[:]), nil +} + +func generateDirHash(directoryPath string) (string, error) { + tarData, err := tarDir(directoryPath) + if err != nil { + return "", fmt.Errorf("could not generate output checksum: %s", err) + } + + checksum := sha1.Sum(tarData) + return hex.EncodeToString(checksum[:]), nil +} + +func tarDir(directoryPath string) ([]byte, error) { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + writeFile := func(p string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + var header *tar.Header + var file *os.File + + header, err = tar.FileInfoHeader(f, f.Name()) + if err != nil { + return err + } + relPath, _ := filepath.Rel(directoryPath, p) + header.Name = relPath + + if err := tw.WriteHeader(header); err != nil { + return err + } + + if f.IsDir() { + return nil + } + + file, err = os.Open(p) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(tw, file) + return err + } + + if err := filepath.Walk(directoryPath, writeFile); err != nil { + return []byte{}, err + } + if err := tw.Flush(); err != nil { + return []byte{}, err + } + + return buf.Bytes(), nil +} diff --git a/builtin/providers/template/resource_template_dir_test.go b/builtin/providers/template/resource_template_dir_test.go new file mode 100644 index 000000000..716a5f0af --- /dev/null +++ b/builtin/providers/template/resource_template_dir_test.go @@ -0,0 +1,104 @@ +package template + +import ( + "fmt" + "testing" + + "errors" + r "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" + "io/ioutil" + "os" + "path/filepath" +) + +const templateDirRenderingConfig = ` +resource "template_dir" "dir" { + source_dir = "%s" + destination_dir = "%s" + vars = %s +}` + +type testTemplate struct { + template string + want string +} + +func testTemplateDirWriteFiles(files map[string]testTemplate) (in, out string, err error) { + in, err = ioutil.TempDir(os.TempDir(), "terraform_template_dir") + if err != nil { + return + } + + for name, file := range files { + path := filepath.Join(in, name) + + err = os.MkdirAll(filepath.Dir(path), 0777) + if err != nil { + return + } + + err = ioutil.WriteFile(path, []byte(file.template), 0777) + if err != nil { + return + } + } + + out = fmt.Sprintf("%s.out", in) + return +} + +func TestTemplateDirRendering(t *testing.T) { + var cases = []struct { + vars string + files map[string]testTemplate + }{ + { + files: map[string]testTemplate{ + "foo.txt": {"${bar}", "bar"}, + "nested/monkey.txt": {"ooh-ooh-ooh-eee-eee", "ooh-ooh-ooh-eee-eee"}, + "maths.txt": {"${1+2+3}", "6"}, + }, + vars: `{bar = "bar"}`, + }, + } + + for _, tt := range cases { + // Write the desired templates in a temporary directory. + in, out, err := testTemplateDirWriteFiles(tt.files) + if err != nil { + t.Skipf("could not write templates to temporary directory: %s", err) + continue + } + defer os.RemoveAll(in) + defer os.RemoveAll(out) + + // Run test case. + r.UnitTest(t, r.TestCase{ + Providers: testProviders, + Steps: []r.TestStep{ + { + Config: fmt.Sprintf(templateDirRenderingConfig, in, out, tt.vars), + Check: func(s *terraform.State) error { + for name, file := range tt.files { + content, err := ioutil.ReadFile(filepath.Join(out, name)) + if err != nil { + return fmt.Errorf("template:\n%s\nvars:\n%s\ngot:\n%s\nwant:\n%s\n", file.template, tt.vars, err, file.want) + } + if string(content) != file.want { + return fmt.Errorf("template:\n%s\nvars:\n%s\ngot:\n%s\nwant:\n%s\n", file.template, tt.vars, content, file.want) + } + } + return nil + }, + }, + }, + CheckDestroy: func(*terraform.State) error { + if _, err := os.Stat(out); os.IsNotExist(err) { + return nil + } + return errors.New("template_dir did not get destroyed") + }, + }) + } +} diff --git a/website/source/docs/providers/template/r/dir.html.md b/website/source/docs/providers/template/r/dir.html.md new file mode 100644 index 000000000..7e0c03067 --- /dev/null +++ b/website/source/docs/providers/template/r/dir.html.md @@ -0,0 +1,58 @@ +--- +layout: "template" +page_title: "Template: template_dir" +sidebar_current: "docs-template-resource-dir" +description: |- + Renders templates from a directory. +--- + +# template_dir + +Renders templates from a directory. + +## Example Usage +```hcl +data "template_directory" "init" { + source_dir = "${path.cwd}/templates" + destination_dir = "${path.cwd}/templates.generated" + + vars { + consul_address = "${aws_instance.consul.private_ip}" + } +} +``` + +## Argument Reference + +The following arguments are supported: + +* `source_path` - (Required) Path to the directory where the files to template reside. + +* `destination_path` - (Required) Path to the directory where the templated files will be written. + +* `vars` - (Optional) Variables for interpolation within the template. Note + that variables must all be primitives. Direct references to lists or maps + will cause a validation error. + +NOTE: Any required parent directories are created automatically. Additionally, any external modification to either the files in the source or destination directories will trigger the resource to be re-created. + +## Template Syntax + +The syntax of the template files is the same as +[standard interpolation syntax](/docs/configuration/interpolation.html), +but you only have access to the variables defined in the `vars` section. + +To access interpolations that are normally available to Terraform +configuration (such as other variables, resource attributes, module +outputs, etc.) you'll have to expose them via `vars` as shown below: + +```hcl +resource "template_dir" "init" { + # ... + + vars { + foo = "${var.foo}" + attr = "${aws_instance.foo.private_ip}" + } +} +``` \ No newline at end of file diff --git a/website/source/layouts/template.erb b/website/source/layouts/template.erb index 8416a3dc8..045e95811 100644 --- a/website/source/layouts/template.erb +++ b/website/source/layouts/template.erb @@ -21,6 +21,15 @@ + + > + Resources + + <% end %> From eda2550074e0b7c8aa1a2b5cd1ba94daf5584903 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 25 Apr 2017 10:03:09 -0700 Subject: [PATCH 2/4] provider/template: template_dir: don't crash if source dir nonexistent When an error is passed, the FileInfo can be nil, which was previously causing a crash on trying to evaluate f.IsDir(). By checking for an error first we avoid this crash. --- builtin/providers/template/resource_template_dir.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builtin/providers/template/resource_template_dir.go b/builtin/providers/template/resource_template_dir.go index 583926bb0..9154d2a89 100644 --- a/builtin/providers/template/resource_template_dir.go +++ b/builtin/providers/template/resource_template_dir.go @@ -89,12 +89,12 @@ func resourceTemplateDirCreate(d *schema.ResourceData, meta interface{}) error { // Recursively crawl the input files/directories and generate the output ones. err := filepath.Walk(sourceDir, func(p string, f os.FileInfo, err error) error { - if f.IsDir() { - return nil - } if err != nil { return err } + if f.IsDir() { + return nil + } relPath, _ := filepath.Rel(sourceDir, p) return generateDirFile(p, path.Join(destinationDir, relPath), f, vars) From eaac9fbca3aebd1d1f23ff2b70089739872b166a Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 25 Apr 2017 10:20:21 -0700 Subject: [PATCH 3/4] provider/template: template_dir explicitly create dest dir Previously we were letting it get implicitly created as part of making the structure for copying in each file, but that isn't sufficient if the source directory is empty. By explicitly creating the directory first we ensure that it will complete successfully even in the case of an empty directory. --- builtin/providers/template/resource_template_dir.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/builtin/providers/template/resource_template_dir.go b/builtin/providers/template/resource_template_dir.go index 9154d2a89..63a2f18dc 100644 --- a/builtin/providers/template/resource_template_dir.go +++ b/builtin/providers/template/resource_template_dir.go @@ -87,11 +87,20 @@ func resourceTemplateDirCreate(d *schema.ResourceData, meta interface{}) error { return err } + // Create the destination directory and any other intermediate directories + // leading to it. + if _, err := os.Stat(destinationDir); err != nil { + if err := os.MkdirAll(destinationDir, 0777); err != nil { + return err + } + } + // Recursively crawl the input files/directories and generate the output ones. err := filepath.Walk(sourceDir, func(p string, f os.FileInfo, err error) error { if err != nil { return err } + if f.IsDir() { return nil } From da49171b0645ce3dcd771fdf2eb6337710447718 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 25 Apr 2017 10:46:55 -0700 Subject: [PATCH 4/4] website: flesh out docs for template_dir resource --- .../docs/providers/template/r/dir.html.md | 80 ++++++++++++++++--- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/website/source/docs/providers/template/r/dir.html.md b/website/source/docs/providers/template/r/dir.html.md index 7e0c03067..9ce700826 100644 --- a/website/source/docs/providers/template/r/dir.html.md +++ b/website/source/docs/providers/template/r/dir.html.md @@ -3,38 +3,87 @@ layout: "template" page_title: "Template: template_dir" sidebar_current: "docs-template-resource-dir" description: |- - Renders templates from a directory. + Renders a directory of templates. --- # template_dir -Renders templates from a directory. +Renders a directory containing templates into a separate directory of +corresponding rendered files. + +`template_dir` is similar to [`template_file`](../d/file.html) but it walks +a given source directory and treats every file it encounters as a template, +rendering it to a corresponding file in the destination directory. + +~> **Note** When working with local files, Terraform will detect the resource +as having been deleted each time a configuration is applied on a new machine +where the destination dir is not present and will generate a diff to create +it. This may cause "noise" in diffs in environments where configurations are +routinely applied by many different users or within automation systems. ## Example Usage + +The following example shows how one might use this resource to produce a +directory of configuration files to upload to a compute instance, using +Amazon EC2 as a placeholder. + ```hcl -data "template_directory" "init" { - source_dir = "${path.cwd}/templates" - destination_dir = "${path.cwd}/templates.generated" +resource "template_dir" "config" { + source_dir = "${path.module}/instance_config_templates" + destination_dir = "${path.cwd}/instance_config" vars { - consul_address = "${aws_instance.consul.private_ip}" + consul_addr = "${var.consul_addr}" } } + +resource "aws_instance" "server" { + ami = "${var.server_ami}" + instance_type = "t2.micro" + + connection { + # ...connection configuration... + } + + provisioner "file" { + # Referencing the template_dir resource ensures that it will be + # created or updated before this aws_instance resource is provisioned. + source = "${template_dir.config.destination_dir}" + destination = "/etc/myapp" + } +} + +variable "consul_addr" {} + +variable "server_ami" {} ``` ## Argument Reference The following arguments are supported: -* `source_path` - (Required) Path to the directory where the files to template reside. +* `source_dir` - (Required) Path to the directory where the files to template reside. -* `destination_path` - (Required) Path to the directory where the templated files will be written. +* `destination_dir` - (Required) Path to the directory where the templated files will be written. * `vars` - (Optional) Variables for interpolation within the template. Note that variables must all be primitives. Direct references to lists or maps will cause a validation error. -NOTE: Any required parent directories are created automatically. Additionally, any external modification to either the files in the source or destination directories will trigger the resource to be re-created. +Any required parent directories of `destination_dir` will be created +automatically, and any pre-existing file or directory at that location will +be deleted before template rendering begins. + +After rendering this resource remembers the content of both the source and +destination directories in the Terraform state, and will plan to recreate the +output directory if any changes are detected during the plan phase. + +Note that it is _not_ safe to use the `file` interpolation function to read +files create by this resource, since that function can be evaluated before the +destination directory has been created or updated. It *is* safe to use the +generated files with resources that directly take filenames as arguments, +as long as the path is constructed using the `destination_dir` attribute +to create a dependency relationship with the `template_dir` resource. ## Template Syntax @@ -44,7 +93,7 @@ but you only have access to the variables defined in the `vars` section. To access interpolations that are normally available to Terraform configuration (such as other variables, resource attributes, module -outputs, etc.) you'll have to expose them via `vars` as shown below: +outputs, etc.) you can expose them via `vars` as shown below: ```hcl resource "template_dir" "init" { @@ -55,4 +104,13 @@ resource "template_dir" "init" { attr = "${aws_instance.foo.private_ip}" } } -``` \ No newline at end of file +``` + +## Attributes + +This resource exports the following attributes: + +* `destination_dir` - The destination directory given in configuration. + Interpolate this attribute into other resource configurations to create + a dependency to ensure that the destination directory is populated before + another resource attempts to read it.