package module import ( "encoding/json" "fmt" "io/ioutil" "log" "os" "path/filepath" "reflect" getter "github.com/hashicorp/go-getter" "github.com/hashicorp/terraform/registry/regsrc" ) const manifestName = "modules.json" // moduleManifest is the serialization structure used to record the stored // module's metadata. type moduleManifest struct { Modules []moduleRecord } // moduleRecords represents the stored module's metadata. // This is compared for equality using '==', so all fields needs to remain // comparable. type moduleRecord struct { // Source is the module source string from the config, minus any // subdirectory. Source string // Key is the locally unique identifier for this module. Key string // Version is the exact version string for the stored module. Version string // Dir is the directory name returned by the FileStorage. This is what // allows us to correlate a particular module version with the location on // disk. Dir string // Root is the root directory containing the module. If the module is // unpacked from an archive, and not located in the root directory, this is // used to direct the loader to the correct subdirectory. This is // independent from any subdirectory in the original source string, which // may traverse further into the module tree. Root string // url is the location of the module source url string // Registry is true if this module is sourced from a registry registry bool } // moduleStorage implements methods to record and fetch metadata about the // modules that have been fetched and stored locally. The getter.Storgae // abstraction doesn't provide the information needed to know which versions of // a module have been stored, or their location. type moduleStorage struct { getter.Storage storageDir string mode GetMode } func newModuleStorage(s getter.Storage, mode GetMode) moduleStorage { return moduleStorage{ Storage: s, storageDir: storageDir(s), mode: mode, } } // The Tree needs to know where to store the module manifest. // Th Storage abstraction doesn't provide access to the storage root directory, // so we extract it here. func storageDir(s getter.Storage) string { // get the StorageDir directly if possible switch t := s.(type) { case *getter.FolderStorage: return t.StorageDir case moduleStorage: return t.storageDir case nil: return "" } // this should be our UI wrapper which is exported here, so we need to // extract the FolderStorage via reflection. fs := reflect.ValueOf(s).Elem().FieldByName("Storage").Interface() return storageDir(fs.(getter.Storage)) } // loadManifest returns the moduleManifest file from the parent directory. func (m moduleStorage) loadManifest() (moduleManifest, error) { manifest := moduleManifest{} manifestPath := filepath.Join(m.storageDir, manifestName) data, err := ioutil.ReadFile(manifestPath) if err != nil && !os.IsNotExist(err) { return manifest, err } if len(data) == 0 { return manifest, nil } if err := json.Unmarshal(data, &manifest); err != nil { return manifest, err } return manifest, nil } // Store the location of the module, along with the version used and the module // root directory. The storage method loads the entire file and rewrites it // each time. This is only done a few times during init, so efficiency is // not a concern. func (m moduleStorage) recordModule(rec moduleRecord) error { manifest, err := m.loadManifest() if err != nil { // if there was a problem with the file, we will attempt to write a new // one. Any non-data related error should surface there. log.Printf("[WARN] error reading module manifest: %s", err) } // do nothing if we already have the exact module for i, stored := range manifest.Modules { if rec == stored { return nil } // they are not equal, but if the storage path is the same we need to // remove this rec to be replaced. if rec.Dir == stored.Dir { manifest.Modules[i] = manifest.Modules[len(manifest.Modules)-1] manifest.Modules = manifest.Modules[:len(manifest.Modules)-1] break } } manifest.Modules = append(manifest.Modules, rec) js, err := json.Marshal(manifest) if err != nil { panic(err) } manifestPath := filepath.Join(m.storageDir, manifestName) return ioutil.WriteFile(manifestPath, js, 0644) } // load the manifest from dir, and return all module versions matching the // provided source. Records with no version info will be skipped, as they need // to be uniquely identified by other means. func (m moduleStorage) moduleVersions(source string) ([]moduleRecord, error) { manifest, err := m.loadManifest() if err != nil { return manifest.Modules, err } var matching []moduleRecord for _, m := range manifest.Modules { if m.Source == source && m.Version != "" { matching = append(matching, m) } } return matching, nil } func (m moduleStorage) moduleDir(key string) (string, error) { manifest, err := m.loadManifest() if err != nil { return "", err } for _, m := range manifest.Modules { if m.Key == key { return m.Dir, nil } } return "", nil } // return only the root directory of the module stored in dir. func (m moduleStorage) getModuleRoot(dir string) (string, error) { manifest, err := m.loadManifest() if err != nil { return "", err } for _, mod := range manifest.Modules { if mod.Dir == dir { return mod.Root, nil } } return "", nil } // record only the Root directory for the module stored at dir. // TODO: remove this compatibility function to store the full moduleRecord. func (m moduleStorage) recordModuleRoot(dir, root string) error { rec := moduleRecord{ Dir: dir, Root: root, } return m.recordModule(rec) } func (m moduleStorage) getStorage(key string, src string) (string, bool, error) { // Get the module with the level specified if we were told to. if m.mode > GetModeNone { log.Printf("[DEBUG] fetching %q with key %q", src, key) if err := m.Storage.Get(key, src, m.mode == GetModeUpdate); err != nil { return "", false, err } } // Get the directory where the module is. dir, found, err := m.Storage.Dir(key) log.Printf("[DEBUG] found %q in %q: %t", src, dir, found) return dir, found, err } // find a stored module that's not from a registry func (m moduleStorage) findModule(key string) (string, error) { if m.mode == GetModeUpdate { return "", nil } return m.moduleDir(key) } // find a registry module func (m moduleStorage) findRegistryModule(mSource, constraint string) (moduleRecord, error) { rec := moduleRecord{ Source: mSource, } // detect if we have a registry source mod, err := regsrc.ParseModuleSource(mSource) switch err { case nil: //ok case regsrc.ErrInvalidModuleSource: return rec, nil default: return rec, err } rec.registry = true log.Printf("[TRACE] %q is a registry module", mod.Module()) versions, err := m.moduleVersions(mod.String()) if err != nil { log.Println("[ERROR] error looking up versions for %q: %s", mod.Module(), err) return rec, err } match, err := newestRecord(versions, constraint) if err != nil { // TODO: does this allow previously unversioned modules? log.Printf("[INFO] no matching version for %q<%s>, %s", mod.Module(), constraint, err) } rec.Dir = match.Dir rec.Version = match.Version found := rec.Dir != "" // we need to lookup available versions // Only on Get if it's not found, on unconditionally on Update if (m.mode == GetModeGet && !found) || (m.mode == GetModeUpdate) { resp, err := lookupModuleVersions(nil, mod) if err != nil { return rec, err } if len(resp.Modules) == 0 { return rec, fmt.Errorf("module %q not found in registry", mod.Module()) } match, err := newestVersion(resp.Modules[0].Versions, constraint) if err != nil { return rec, err } if match == nil { return rec, fmt.Errorf("no versions for %q found matching %q", mod.Module(), constraint) } rec.Version = match.Version rec.url, err = lookupModuleLocation(nil, mod, rec.Version) if err != nil { return rec, err } } return rec, nil }