From 67ca0679103ff7fbf0ee866deedb1d740537468a Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 11 Mar 2020 18:00:43 -0700 Subject: [PATCH] internal/providercache: Linking from one cache to another When a system-wide shared plugin cache is configured, we'll want to make use of entries already in the shared cache when populating a local (configuration-specific) cache. This new method LinkFromOtherCache encapsulates the work of placing a link from one cache to another. If possible it will create a symlink, therefore retaining a key advantage of configuring a shared plugin cache, but otherwise we'll do a deep copy of the package directory from one cache to the other. Our old provider installer would always skip trying to create symlinks on Windows because Go standard library support for os.Symlink on Windows was inconsistent in older versions. However, os.Symlink can now create symlinks using a new API introduced in a Windows 10 update and cleanly fail if symlink creation is impossible, so it's safe for us to just try to create the symlink and react if that produces an error, just as we used to do on non-Windows systems when possibly creating symlinks on filesystems that cannot support them. --- internal/providercache/dir_modify.go | 95 ++++++++++++++++++++++ internal/providercache/dir_modify_test.go | 98 +++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 internal/providercache/dir_modify.go create mode 100644 internal/providercache/dir_modify_test.go diff --git a/internal/providercache/dir_modify.go b/internal/providercache/dir_modify.go new file mode 100644 index 000000000..c43af2dad --- /dev/null +++ b/internal/providercache/dir_modify.go @@ -0,0 +1,95 @@ +package providercache + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hashicorp/terraform/internal/copydir" + "github.com/hashicorp/terraform/internal/getproviders" +) + +// LinkFromOtherCache takes a CachedProvider value produced from another Dir +// and links it into the cache represented by the receiver Dir. +// +// This is used to implement tiered caching, where new providers are first +// populated into a system-wide shared cache and then linked from there into +// a configuration-specific local cache. +// +// It's invalid to link a CachedProvider from a particular Dir into that same +// Dir, because that would otherwise potentially replace a real package +// directory with a circular link back to itself. +func (d *Dir) LinkFromOtherCache(entry *CachedProvider) error { + newPath := getproviders.UnpackedDirectoryPathForPackage( + d.baseDir, entry.Provider, entry.Version, d.targetPlatform, + ) + currentPath := entry.PackageDir + + absNew, err := filepath.Abs(newPath) + if err != nil { + return fmt.Errorf("failed to make new path %s absolute: %s", newPath, err) + } + absCurrent, err := filepath.Abs(currentPath) + if err != nil { + return fmt.Errorf("failed to make existing cache path %s absolute: %s", currentPath, err) + } + + // Before we do anything else, we'll do a quick check to make sure that + // these two paths are not pointing at the same physical directory on + // disk. This compares the files by their OS-level device and directory + // entry identifiers, not by their virtual filesystem paths. + if same, err := copydir.SameFile(absNew, absCurrent); same { + return fmt.Errorf("cannot link existing cache path %s to itself", newPath) + } else if err != nil { + return fmt.Errorf("failed to determine if %s and %s are the same: %s", currentPath, newPath, err) + } + + // Invalidate our metaCache so that subsequent read calls will re-scan to + // incorporate any changes we make here. + d.metaCache = nil + + // Delete anything that's already present at this path first. + err = os.RemoveAll(currentPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove existing %s before linking it to %s: %s", currentPath, newPath, err) + } + + // We'll prefer to create a symlink if possible, but we'll fall back to + // a recursive copy if symlink creation fails. It could fail for a number + // of reasons, including being on Windows 8 without administrator + // privileges or being on a legacy filesystem like FAT that has no way + // to represent a symlink. (Generalized symlink support for Windows was + // introduced in a Windows 10 minor update.) + // + // We'd prefer to use a relative path for the symlink to reduce the risk + // of it being broken by moving things around later, but we'll fall back + // on the absolute path we already calculated if that isn't possible + // (e.g. because the two paths are on different "volumes" on an OS with + // that concept, like Windows with drive letters and UNC host/share names.) + linkTarget, err := filepath.Rel(newPath, absCurrent) + if err != nil { + linkTarget = absCurrent + } + + parentDir := filepath.Dir(absNew) + err = os.MkdirAll(parentDir, 0755) + if err != nil && os.IsExist(err) { + return fmt.Errorf("failed to create parent directories leading to %s: %s", newPath, err) + } + + err = os.Symlink(linkTarget, absNew) + if err == nil { + // Success, then! + return nil + } + + // If we get down here then symlinking failed and we need a deep copy + // instead. + err = copydir.CopyDir(absNew, absCurrent) + if err != nil { + return fmt.Errorf("failed to either symlink or copy %s to %s: %s", absCurrent, absNew, err) + } + + // If we got here then apparently our copy succeeded, so we're done. + return nil +} diff --git a/internal/providercache/dir_modify_test.go b/internal/providercache/dir_modify_test.go new file mode 100644 index 000000000..fcfcf72ca --- /dev/null +++ b/internal/providercache/dir_modify_test.go @@ -0,0 +1,98 @@ +package providercache + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/apparentlymart/go-versions/versions" + "github.com/google/go-cmp/cmp" + + "github.com/hashicorp/terraform/addrs" + "github.com/hashicorp/terraform/internal/getproviders" +) + +func LinkFromOtherCache(t *testing.T) { + srcDirPath := "testdata/cachedir" + tmpDirPath, err := ioutil.TempDir("", "terraform-test-providercache") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDirPath) + + windowsPlatform := getproviders.Platform{ + OS: "windows", + Arch: "amd64", + } + nullProvider := addrs.NewProvider( + addrs.DefaultRegistryHost, "hashicorp", "null", + ) + + srcDir := newDirWithPlatform(srcDirPath, windowsPlatform) + tmpDir := newDirWithPlatform(tmpDirPath, windowsPlatform) + + // First we'll check our preconditions: srcDir should have only the + // null provider version 2.0.0 in it, because we're faking that we're on + // Windows, and tmpDir should have no providers in it at all. + + gotSrcInitial := srcDir.AllAvailablePackages() + wantSrcInitial := map[addrs.Provider][]CachedProvider{ + nullProvider: { + CachedProvider{ + Provider: nullProvider, + + // We want 2.0.0 rather than 2.1.0 because the 2.1.0 package is + // still packed and thus not considered to be a cache member. + Version: versions.MustParseVersion("2.0.0"), + + PackageDir: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64", + ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe", + }, + }, + } + if diff := cmp.Diff(wantSrcInitial, gotSrcInitial); diff != "" { + t.Fatalf("incorrect initial source directory contents\n%s", diff) + } + + gotTmpInitial := tmpDir.AllAvailablePackages() + wantTmpInitial := map[addrs.Provider][]CachedProvider{} + if diff := cmp.Diff(wantTmpInitial, gotTmpInitial); diff != "" { + t.Fatalf("incorrect initial temp directory contents\n%s", diff) + } + + cacheEntry := srcDir.ProviderLatestVersion(nullProvider) + if cacheEntry == nil { + // This is weird because we just checked for the presence of this above + t.Fatalf("null provider has no latest version in source directory") + } + + err = tmpDir.LinkFromOtherCache(cacheEntry) + if err != nil { + t.Fatalf("LinkFromOtherCache failed: %s", err) + } + + // Now we should see the same version reflected in the temporary directory. + got := tmpDir.AllAvailablePackages() + want := map[addrs.Provider][]CachedProvider{ + nullProvider: { + CachedProvider{ + Provider: nullProvider, + + // We want 2.0.0 rather than 2.1.0 because the 2.1.0 package is + // still packed and thus not considered to be a cache member. + Version: versions.MustParseVersion("2.0.0"), + + // These are still pointed into the testdata directory because + // we created a symlink in our tmpDir. (This part of the test + // is expected to fail if the temporary directory is on a + // filesystem that cannot support symlinks, in which case + // we should see the temporary directory paths here instead.) + PackageDir: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64", + ExecutableFile: "testdata/cachedir/registry.terraform.io/hashicorp/null/2.0.0/windows_amd64/terraform-provider-null.exe", + }, + }, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("wrong cache contents after link\n%s", diff) + } +}