update the module manifest implementation

The getter.Storage abstraction is proving entirely inadequate here, but
we can't replace it wholesale at the moment.

The Tree loader needs to know the location of the manifest before it can
start loading any modules. Since the version will have to be part of the
hashed storage key, there is no way to know what version of each module
are stored. The storageDir function will extract the StorageDir field
from the underlying FolderStorage instance for the tree to locate the
manifest.

config/module/storage.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 )
 
+const manifestName = "modules.json"
+
 // moduleManifest is the serialization structure used to record the stored
 // module's metadata.
 type moduleManifest struct {
@@ -39,21 +41,19 @@ type moduleRecord struct {
 	Root string
 }
 
-// Return the path to the manifest in parent of the storage directory dir.
-func moduleManifestPath(dir string) string {
-	const filename = "modules.json"
-	// Get the parent directory.
-	// The current FolderStorage implementation needed to be able to create
-	// this directory, so we can be reasonably certain we can use it.
-	parent := filepath.Dir(filepath.Clean(dir))
-	return filepath.Join(parent, filename)
+// moduleStorgae implements methods to record and fetch metadata about the
+// modules that have been fetched and stored locally. The getter.Storgae
+// abstraction doesn't provide the information needed to know which versions of
+// a module have been stored, or their location.
+type moduleStorage struct {
+	storageDir string
 }
 
 // loadManifest returns the moduleManifest file from the parent directory.
-func loadManifest(dir string) (moduleManifest, error) {
+func (m moduleStorage) loadManifest() (moduleManifest, error) {
 	manifest := moduleManifest{}
 
-	manifestPath := moduleManifestPath(dir)
+	manifestPath := filepath.Join(m.storageDir, manifestName)
 	data, err := ioutil.ReadFile(manifestPath)
 	if err != nil && !os.IsNotExist(err) {
 		return manifest, err
@@ -73,50 +73,50 @@ func loadManifest(dir string) (moduleManifest, error) {
 // root directory. The storage method loads the entire file and rewrites it
 // each time. This is only done a few times during init, so efficiency is
 // not a concern.
-func recordModule(dir string, m moduleRecord) error {
-	manifest, err := loadManifest(dir)
+func (m moduleStorage) recordModule(rec moduleRecord) error {
+	manifest, err := m.loadManifest()
 	if err != nil {
 		// if there was a problem with the file, we will attempt to write a new
 		// one. Any non-data related error should surface there.
-		log.Printf("[WARN] error reading module manifest from %q: %s", dir, err)
+		log.Printf("[WARN] error reading module manifest: %s", err)
 	}
 
 	// do nothing if we already have the exact module
 	for i, stored := range manifest.Modules {
-		if m == stored {
+		if rec == stored {
 			return nil
 		}
 
 		// they are not equal, but if the storage path is the same we need to
-		// remove this record to be replaced.
-		if m.Dir == stored.Dir {
+		// remove this rec to be replaced.
+		if rec.Dir == stored.Dir {
 			manifest.Modules[i] = manifest.Modules[len(manifest.Modules)-1]
 			manifest.Modules = manifest.Modules[:len(manifest.Modules)-1]
 			break
 		}
 	}
 
-	manifest.Modules = append(manifest.Modules, m)
+	manifest.Modules = append(manifest.Modules, rec)
 
 	js, err := json.Marshal(manifest)
 	if err != nil {
 		panic(err)
 	}
 
-	manifestPath := moduleManifestPath(dir)
+	manifestPath := filepath.Join(m.storageDir, manifestName)
 	return ioutil.WriteFile(manifestPath, js, 0644)
 }
 
 // return only the root directory of the module stored in dir.
-func getModuleRoot(dir string) (string, error) {
-	manifest, err := loadManifest(dir)
+func (m moduleStorage) getModuleRoot(dir string) (string, error) {
+	manifest, err := m.loadManifest()
 	if err != nil {
 		return "", err
 	}
 
-	for _, m := range manifest.Modules {
-		if m.Dir == dir {
-			return m.Root, nil
+	for _, mod := range manifest.Modules {
+		if mod.Dir == dir {
+			return mod.Root, nil
 		}
 	}
 	return "", nil
@@ -124,11 +124,11 @@ func getModuleRoot(dir string) (string, error) {
 
 // record only the Root directory for the module stored at dir.
 // TODO: remove this compatibility function to store the full moduleRecord.
-func recordModuleRoot(dir, root string) error {
-	m := moduleRecord{
+func (m moduleStorage) recordModuleRoot(dir, root string) error {
+	rec := moduleRecord{
 		Dir:  dir,
 		Root: root,
 	}
 
-	return recordModule(dir, m)
+	return m.recordModule(rec)
 }

config/module/tree.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"sync"
 
@@ -27,6 +28,7 @@ type Tree struct {
 	children map[string]*Tree
 	path     []string
 	lock     sync.RWMutex
+	storage  string
 }
 
 // NewTree returns a new Tree for the given config structure.
@@ -162,6 +164,9 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
 	t.lock.Lock()
 	defer t.lock.Unlock()
 
+	// discover where our modules are going to be stored
+	storage := moduleStorage{storageDir: storageDir(s)}
+
 	// Reset the children if we have any
 	t.children = nil
 
@@ -206,7 +211,7 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
 		// In order to load the Tree we need to find out if there was another
 		// subDir stored from discovery.
 		if found && mode != GetModeUpdate {
-			subDir, err := getModuleRoot(dir)
+			subDir, err := storage.getModuleRoot(dir)
 			if err != nil {
 				// If there's a problem with the subdir record, we'll let the
 				// recordSubdir method fix it up.  Any other errors filesystem
@@ -268,7 +273,7 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
 
 			subDir = fullDir[len(dir)+1:]
 
-			if err := recordModuleRoot(dir, subDir); err != nil {
+			if err := storage.recordModuleRoot(dir, subDir); err != nil {
 				return err
 			}
 			dir = fullDir
@@ -645,3 +650,19 @@ func (e *treeError) Error() string {
 
 	return out.String()
 }
+
+// The Tree needs to know where to store the module manifest.
+// Th Storage abstraction doesn't provide access to the storage root directory,
+// so we extract it here.
+// TODO: This needs to be replaced by refactoring the getter.Storage usage for
+//       modules.
+func storageDir(s getter.Storage) string {
+	// get the StorageDir directly if we have a FolderStorage
+	if s, ok := s.(*getter.FolderStorage); ok {
+		return s.StorageDir
+	}
+
+	// this is our UI wrapper, so we need to extract the FolderStorage
+	fs := reflect.ValueOf(s).Elem().FieldByName("Storage").Interface()
+	return storageDir(fs.(getter.Storage))
+}

config/module/tree_test.go

@@ -271,15 +271,17 @@ func TestTree_recordManifest(t *testing.T) {
 	}
 	defer os.RemoveAll(td)
 
+	storage := moduleStorage{storageDir: td}
+
 	dir := filepath.Join(td, "0131bf0fef686e090b16bdbab4910ddf")
 
 	subDir := "subDirName"
 
 	// record and read the subdir path
-	if err := recordModuleRoot(dir, subDir); err != nil {
+	if err := storage.recordModuleRoot(dir, subDir); err != nil {
 		t.Fatal(err)
 	}
-	actual, err := getModuleRoot(dir)
+	actual, err := storage.getModuleRoot(dir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -290,10 +292,10 @@ func TestTree_recordManifest(t *testing.T) {
 
 	// overwrite the path, and nmake sure we get the new one
 	subDir = "newSubDir"
-	if err := recordModuleRoot(dir, subDir); err != nil {
+	if err := storage.recordModuleRoot(dir, subDir); err != nil {
 		t.Fatal(err)
 	}
-	actual, err = getModuleRoot(dir)
+	actual, err = storage.getModuleRoot(dir)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -303,21 +305,21 @@ func TestTree_recordManifest(t *testing.T) {
 	}
 
 	// create a fake entry
-	if err := ioutil.WriteFile(moduleManifestPath(dir), []byte("BAD DATA"), 0644); err != nil {
+	if err := ioutil.WriteFile(filepath.Join(td, manifestName), []byte("BAD DATA"), 0644); err != nil {
 		t.Fatal(err)
 	}
 
 	// this should fail because there aare now 2 entries
-	actual, err = getModuleRoot(dir)
+	actual, err = storage.getModuleRoot(dir)
 	if err == nil {
 		t.Fatal("expected multiple subdir entries")
 	}
 
 	// writing the subdir entry should remove the incorrect value
-	if err := recordModuleRoot(dir, subDir); err != nil {
+	if err := storage.recordModuleRoot(dir, subDir); err != nil {
 		t.Fatal(err)
 	}
-	actual, err = getModuleRoot(dir)
+	actual, err = storage.getModuleRoot(dir)
 	if err != nil {
 		t.Fatal(err)
 	}