internal/linker: place files under GARBLE_CACHE

This means we now have a unified cache directory for garble, which is now documented in the README. I considered using the same hash-based cache used for pkgCache, but decided against it since that cache implementation only stores regular files without any executable bits set. We could force the cache package to do what we want here, but I'm leaning against it for now given that single files work OK.
2 years ago · 6dd5c53a91
parent 79376a15f9
commit 6dd5c53a91
6 changed files with 45 additions and 46 deletions
--- a/README.md
+++ b/README.md
@ -93,6 +93,10 @@ Note that the first call to `garble build` may be comparatively slow,
 as it has to obfuscate each package for the first time. This is akin to clearing
 `GOCACHE` with `go clean -cache` and running a `go build` from scratch.

+Garble also makes use of its own cache to reuse work, akin to Go's `GOCACHE`.
+It defaults to a directory under your user's cache directory,
+such as `~/.cache/garble`, and can be placed elsewhere by setting `GARBLE_CACHE`.
+
 ### Determinism and seeds

 Just like Go, garble builds are deterministic and reproducible in nature.
--- a/internal/linker/linker.go
+++ b/internal/linker/linker.go
@ -27,10 +27,8 @@ const (
 	TinyEnv        = "GARBLE_LINK_TINY"
 	EntryOffKeyEnv = "GARBLE_LINK_ENTRYOFF_KEY"

-	cacheDirName   = "garble"
-	versionExt     = ".version"
-	garbleCacheDir = "GARBLE_CACHE_DIR"
-	baseSrcSubdir  = "src"
+	versionExt    = ".version"
+	baseSrcSubdir = "src"
 )

 //go:embed patches/*/*.patch
@ -137,25 +135,14 @@ func applyPatches(srcDir, workingDir string, modFiles map[string]bool, patches [
 	return mod, nil
 }

-// TODO: put linker binaries into fsCache in the main package
-
-func cachePath() (string, error) {
-	var cacheDir string
-	if val, ok := os.LookupEnv(garbleCacheDir); ok {
-		cacheDir = val
-	} else {
-		userCacheDir, err := os.UserCacheDir()
-		if err != nil {
-			panic(fmt.Errorf("cannot retreive user cache directory: %v", err))
-		}
-		cacheDir = userCacheDir
-	}
-
-	cacheDir = filepath.Join(cacheDir, cacheDirName)
+func cachePath(cacheDir string) (string, error) {
+	// Use a subdirectory to clarify what we're using it for.
+	// Name it "tool", like Go's pkg/tool, as we might want to rebuild
+	// other Go toolchain programs like the compiler or assembler in the future.
+	cacheDir = filepath.Join(cacheDir, "tool")
 	if err := os.MkdirAll(cacheDir, 0o777); err != nil {
 		return "", err
 	}
-
 	goExe := ""
 	if runtime.GOOS == "windows" {
 		goExe = ".exe"
@ -226,7 +213,7 @@ func buildLinker(workingDir string, overlay map[string]string, outputLinkPath st
 	return nil
 }

-func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) {
+func PatchLinker(goRoot, goVersion, cacheDir, tempDir string) (string, func(), error) {
 	// rxVersion looks for a version like "go1.19" or "go1.20"
 	rxVersion := regexp.MustCompile(`go\d+\.\d+`)
 	majorGoVersion := rxVersion.FindString(goVersion)
@ -236,7 +223,7 @@ func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) {
 		panic(fmt.Errorf("cannot retrieve linker patches: %v", err))
 	}

-	outputLinkPath, err := cachePath()
+	outputLinkPath, err := cachePath(cacheDir)
 	if err != nil {
 		return "", nil, err
 	}
--- a/main.go
+++ b/main.go
@ -400,12 +400,15 @@ func mainErr(args []string) error {
 			if err := os.RemoveAll(os.Getenv("GARBLE_SHARED")); err != nil {
 				fmt.Fprintf(os.Stderr, "could not clean up GARBLE_SHARED: %v\n", err)
 			}
-			fsCache, err := openCache()
-			if err == nil {
-				err = fsCache.Trim()
-			}
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err)
+			// skip the trim if we didn't even start a build
+			if sharedCache != nil {
+				fsCache, err := openCache()
+				if err == nil {
+					err = fsCache.Trim()
+				}
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err)
+				}
 			}
 		}()
 		if err != nil {
@ -455,7 +458,7 @@ func mainErr(args []string) error {

 		executablePath := args[0]
 		if tool == "link" {
-			modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedTempDir)
+			modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedCache.CacheDir, sharedTempDir)
 			if err != nil {
 				return fmt.Errorf("cannot get modified linker: %v", err)
 			}
@ -543,6 +546,20 @@ This command wraps "go %s". Below is its help:
 		return nil, err
 	}

+	// Always an absolute directory; defaults to e.g. "~/.cache/garble".
+	if dir := os.Getenv("GARBLE_CACHE"); dir != "" {
+		sharedCache.CacheDir, err = filepath.Abs(dir)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		parentDir, err := os.UserCacheDir()
+		if err != nil {
+			return nil, err
+		}
+		sharedCache.CacheDir = filepath.Join(parentDir, "garble")
+	}
+
 	binaryBuildID, err := buildidOf(sharedCache.ExecPath)
 	if err != nil {
 		return nil, err
@ -1304,18 +1321,9 @@ func (c *pkgCache) CopyFrom(c2 pkgCache) {
 }

 func openCache() (*cache.Cache, error) {
-	dir := os.Getenv("GARBLE_CACHE") // e.g. "~/.cache/garble"
-	if dir == "" {
-		parentDir, err := os.UserCacheDir()
-		if err != nil {
-			return nil, err
-		}
-		dir = filepath.Join(parentDir, "garble")
-	}
 	// Use a subdirectory for the hashed build cache, to clarify what it is,
 	// and to allow us to have other directories or files later on without mixing.
-	dir = filepath.Join(dir, "build")
-
+	dir := filepath.Join(sharedCache.CacheDir, "build")
 	if err := os.MkdirAll(dir, 0o777); err != nil {
 		return nil, err
 	}
--- a/main_test.go
+++ b/main_test.go
@ -101,11 +101,9 @@ func TestScript(t *testing.T) {
 				// coverage. Otherwise, the coverage info might be incomplete.
 				env.Setenv("GOCACHE", filepath.Join(tempCacheDir, "go-cache"))
 				env.Setenv("GARBLE_CACHE", filepath.Join(tempCacheDir, "garble-cache"))
-				env.Setenv("GARBLE_CACHE_DIR", filepath.Join(tempCacheDir, "garble-cache-2"))
 			} else {
 				// GOCACHE is initialized by gotooltest to use the host's cache.
 				env.Setenv("GARBLE_CACHE", filepath.Join(hostCacheDir, "garble"))
-				env.Setenv("GARBLE_CACHE_DIR", hostCacheDir)
 			}
 			return nil
 		},
--- a/shared.go
+++ b/shared.go
@ -32,6 +32,8 @@ type sharedCacheType struct {
 	ExecPath          string   // absolute path to the garble binary being used
 	ForwardBuildFlags []string // build flags fed to the original "garble ..." command

+	CacheDir string // absolute path to the GARBLE_CACHE directory being used
+
 	// ListedPackages contains data obtained via 'go list -json -export -deps'.
 	// This allows us to obtain the non-obfuscated export data of all dependencies,
 	// useful for type checking of the packages as we obfuscate them.
--- a/testdata/script/linker.txtar
+++ b/testdata/script/linker.txtar
@ -1,7 +1,7 @@
 # Past garble versions might not properly patch cmd/link with "git apply"
 # when running inside a git repository. Skip the extra check with -short.
 [!short] [exec:git] exec git init -q
-[!short] [exec:git] env GARBLE_CACHE_DIR=${WORK}/linker-cache
+[!short] [exec:git] env GARBLE_CACHE=${WORK}/garble-cache

 # Any build settings for the main build shouldn't affect building the linker.
 # If this flag makes it through when using build commands on std or cmd,
@ -20,10 +20,10 @@ exec ./main
 [!windows] env GOOS=windows
 [windows] env GOOS=linux
 exec garble build
-[!windows] [exec:git]   exists ${GARBLE_CACHE_DIR}/garble/link
-[!windows] [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link.exe
-[windows]  [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link
-[windows]  [exec:git]   exists ${GARBLE_CACHE_DIR}/garble/link.exe
+[!windows] [exec:git]   exists ${GARBLE_CACHE}/tool/link
+[!windows] [exec:git] ! exists ${GARBLE_CACHE}/tool/link.exe
+[windows]  [exec:git] ! exists ${GARBLE_CACHE}/tool/link
+[windows]  [exec:git]   exists ${GARBLE_CACHE}/tool/link.exe
 env GOOS=

 # Verify a build without garble.