config/module: start, lots of initial work

config/module/folder_storage.go

@@ -0,0 +1,65 @@
+package module
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// FolderStorage is an implementation of the Storage interface that manages
+// modules on the disk.
+type FolderStorage struct {
+	// StorageDir is the directory where the modules will be stored.
+	StorageDir string
+}
+
+// Dir implements Storage.Dir
+func (s *FolderStorage) Dir(source string) (d string, e bool, err error) {
+	d = s.dir(source)
+	_, err = os.Stat(d)
+	if err == nil {
+		// Directory exists
+		e = true
+		return
+	}
+	if os.IsNotExist(err) {
+		// Directory doesn't exist
+		d = ""
+		e = false
+		err = nil
+		return
+	}
+
+	// An error
+	d = ""
+	e = false
+	return
+}
+
+// Get implements Storage.Get
+func (s *FolderStorage) Get(source string, update bool) error {
+	dir := s.dir(source)
+	if !update {
+		if _, err := os.Stat(dir); err == nil {
+			// If the directory already exists, then we're done since
+			// we're not updating.
+			return nil
+		} else if !os.IsNotExist(err) {
+			// If the error we got wasn't a file-not-exist error, then
+			// something went wrong and we should report it.
+			return fmt.Errorf("Error reading module directory: %s", err)
+		}
+	}
+
+	// Get the source. This always forces an update.
+	return Get(dir, source)
+}
+
+// dir returns the directory name internally that we'll use to map to
+// internally.
+func (s *FolderStorage) dir(source string) string {
+	sum := md5.Sum([]byte(source))
+	return filepath.Join(s.StorageDir, hex.EncodeToString(sum[:]))
+}

config/module/folder_storage_test.go

@@ -0,0 +1,60 @@
+package module
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestFolderStorage_impl(t *testing.T) {
+	var _ Storage = new(FolderStorage)
+}
+
+func TestFolderStorage(t *testing.T) {
+	s := &FolderStorage{StorageDir: tempDir(t)}
+
+	module := testModule("basic")
+
+	// A module shouldn't exist at first...
+	_, ok, err := s.Dir(module)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if ok {
+		t.Fatal("should not exist")
+	}
+
+	// We can get it
+	err = s.Get(module, false)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Now the module exists
+	dir, ok, err := s.Dir(module)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if !ok {
+		t.Fatal("should exist")
+	}
+
+	mainPath := filepath.Join(dir, "main.tf")
+	if _, err := os.Stat(mainPath); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+}
+
+func tempDir(t *testing.T) string {
+	dir, err := ioutil.TempDir("", "tf")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if err := os.RemoveAll(dir); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	return dir
+}

config/module/get.go

@@ -0,0 +1,53 @@
+package module
+
+import (
+	"fmt"
+	"net/url"
+)
+
+// Getter defines the interface that schemes must implement to download
+// and update modules.
+type Getter interface {
+	// Get downloads the given URL into the given directory. This always
+	// assumes that we're updating and gets the latest version that it can.
+	//
+	// The directory may already exist (if we're updating). If it is in a
+	// format that isn't understood, an error should be returned. Get shouldn't
+	// simply nuke the directory.
+	Get(string, *url.URL) error
+}
+
+// Getters is the mapping of scheme to the Getter implementation that will
+// be used to get a dependency.
+var Getters map[string]Getter
+
+func init() {
+	Getters = map[string]Getter{
+		"file": new(FileGetter),
+	}
+}
+
+// Get downloads the module specified by src into the folder specified by
+// dst. If dst already exists, Get will attempt to update it.
+//
+// src is a URL, whereas dst is always just a file path to a folder. This
+// folder doesn't need to exist. It will be created if it doesn't exist.
+func Get(dst, src string) error {
+	u, err := url.Parse(src)
+	if err != nil {
+		return err
+	}
+
+	g, ok := Getters[u.Scheme]
+	if !ok {
+		return fmt.Errorf(
+			"module download not supported for scheme '%s'", u.Scheme)
+	}
+
+	err = g.Get(dst, u)
+	if err != nil {
+		err = fmt.Errorf("error downloading module '%s': %s", src, err)
+	}
+
+	return err
+}

config/module/get_file.go

@@ -0,0 +1,41 @@
+package module
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+// FileGetter is a Getter implementation that will download a module from
+// a file scheme.
+type FileGetter struct{}
+
+func (g *FileGetter) Get(dst string, u *url.URL) error {
+	fi, err := os.Stat(dst)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// If the destination already exists, it must be a symlink
+	if err == nil {
+		mode := fi.Mode()
+		if mode&os.ModeSymlink != 0 {
+			return fmt.Errorf("destination exists and is not a symlink")
+		}
+	}
+
+	// The source path must exist and be a directory to be usable.
+	if fi, err := os.Stat(u.Path); err != nil {
+		return fmt.Errorf("source path error: %s", err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("source path must be a directory")
+	}
+
+	// Create all the parent directories
+	if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		return err
+	}
+
+	return os.Symlink(u.Path, dst)
+}

config/module/module.go

@@ -0,0 +1,7 @@
+package module
+
+// Module represents the metadata for a single module.
+type Module struct {
+	Name   string
+	Source string
+}

config/module/module_test.go

@@ -0,0 +1,21 @@
+package module
+
+import (
+	"net/url"
+	"path/filepath"
+)
+
+const fixtureDir = "./test-fixtures"
+
+func testModule(n string) string {
+	p := filepath.Join(fixtureDir, n)
+	p, err := filepath.Abs(p)
+	if err != nil {
+		panic(err)
+	}
+
+	var url url.URL
+	url.Scheme = "file"
+	url.Path = p
+	return url.String()
+}

config/module/storage.go

@@ -0,0 +1,13 @@
+package module
+
+// Storage is an interface that knows how to lookup downloaded modules
+// as well as download and update modules from their sources into the
+// proper location.
+type Storage interface {
+	// Dir returns the directory on local disk where the modulue source
+	// can be loaded from.
+	Dir(string) (string, bool, error)
+
+	// Get will download and optionally update the given module.
+	Get(string, bool) error
+}

config/module/test-fixtures/basic/main.tf

@@ -0,0 +1 @@
+# Hello

config/module/tree.go

@@ -0,0 +1,71 @@
+package module
+
+import (
+	"github.com/hashicorp/terraform/config"
+)
+
+// Tree represents the module import tree of configurations.
+//
+// This Tree structure can be used to get (download) new modules, load
+// all the modules without getting, flatten the tree into something
+// Terraform can use, etc.
+type Tree struct {
+	Config   *config.Config
+	Children []*Tree
+}
+
+// GetMode is an enum that describes how modules are loaded.
+//
+// GetModeLoad says that modules will not be downloaded or updated, they will
+// only be loaded from the storage.
+//
+// GetModeGet says that modules can be initially downloaded if they don't
+// exist, but otherwise to just load from the current version in storage.
+//
+// GetModeUpdate says that modules should be checked for updates and
+// downloaded prior to loading. If there are no updates, we load the version
+// from disk, otherwise we download first and then load.
+type GetMode byte
+
+const (
+	GetModeNone GetMode = iota
+	GetModeGet
+	GetModeUpdate
+)
+
+// Flatten takes the entire module tree and flattens it into a single
+// namespace in *config.Config with no module imports.
+//
+// Validate is called here implicitly, since it is important that semantic
+// checks pass before flattening the configuration. Otherwise, encapsulation
+// breaks in horrible ways and the errors that come out the other side
+// will be surprising.
+func (t *Tree) Flatten() (*config.Config, error) {
+	return nil, nil
+}
+
+// Modules returns the list of modules that this tree imports.
+func (t *Tree) Modules() []*Module {
+	return nil
+}
+
+// Load loads the configuration of the entire tree.
+//
+// The parameters are used to tell the tree where to find modules and
+// whether it can download/update modules along the way.
+//
+// Various semantic-like checks are made along the way of loading since
+// module trees inherently require the configuration to be in a reasonably
+// sane state: no circular dependencies, proper module sources, etc. A full
+// suite of validations can be done by running Validate (after loading).
+func (t *Tree) Load(s Storage, mode GetMode) error {
+	return nil
+}
+
+// Validate does semantic checks on the entire tree of configurations.
+//
+// This will call the respective config.Config.Validate() functions as well
+// as verifying things such as parameters/outputs between the various modules.
+func (t *Tree) Validate() error {
+	return nil
+}