From 6dd5c53a9134a08c6ea93b5c7c94820d9c183c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 3 Jun 2023 22:28:32 +0100 Subject: [PATCH] internal/linker: place files under GARBLE_CACHE This means we now have a unified cache directory for garble, which is now documented in the README. I considered using the same hash-based cache used for pkgCache, but decided against it since that cache implementation only stores regular files without any executable bits set. We could force the cache package to do what we want here, but I'm leaning against it for now given that single files work OK. --- README.md | 4 ++++ internal/linker/linker.go | 31 ++++++++------------------ main.go | 42 +++++++++++++++++++++--------------- main_test.go | 2 -- shared.go | 2 ++ testdata/script/linker.txtar | 10 ++++----- 6 files changed, 45 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index cd0867f..8059e8d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,10 @@ Note that the first call to `garble build` may be comparatively slow, as it has to obfuscate each package for the first time. This is akin to clearing `GOCACHE` with `go clean -cache` and running a `go build` from scratch. +Garble also makes use of its own cache to reuse work, akin to Go's `GOCACHE`. +It defaults to a directory under your user's cache directory, +such as `~/.cache/garble`, and can be placed elsewhere by setting `GARBLE_CACHE`. + ### Determinism and seeds Just like Go, garble builds are deterministic and reproducible in nature. diff --git a/internal/linker/linker.go b/internal/linker/linker.go index dce2764..f3bb932 100644 --- a/internal/linker/linker.go +++ b/internal/linker/linker.go @@ -27,10 +27,8 @@ const ( TinyEnv = "GARBLE_LINK_TINY" EntryOffKeyEnv = "GARBLE_LINK_ENTRYOFF_KEY" - cacheDirName = "garble" - versionExt = ".version" - garbleCacheDir = "GARBLE_CACHE_DIR" - baseSrcSubdir = "src" + versionExt = ".version" + baseSrcSubdir = "src" ) //go:embed patches/*/*.patch @@ -137,25 +135,14 @@ func applyPatches(srcDir, workingDir string, modFiles map[string]bool, patches [ return mod, nil } -// TODO: put linker binaries into fsCache in the main package - -func cachePath() (string, error) { - var cacheDir string - if val, ok := os.LookupEnv(garbleCacheDir); ok { - cacheDir = val - } else { - userCacheDir, err := os.UserCacheDir() - if err != nil { - panic(fmt.Errorf("cannot retreive user cache directory: %v", err)) - } - cacheDir = userCacheDir - } - - cacheDir = filepath.Join(cacheDir, cacheDirName) +func cachePath(cacheDir string) (string, error) { + // Use a subdirectory to clarify what we're using it for. + // Name it "tool", like Go's pkg/tool, as we might want to rebuild + // other Go toolchain programs like the compiler or assembler in the future. + cacheDir = filepath.Join(cacheDir, "tool") if err := os.MkdirAll(cacheDir, 0o777); err != nil { return "", err } - goExe := "" if runtime.GOOS == "windows" { goExe = ".exe" @@ -226,7 +213,7 @@ func buildLinker(workingDir string, overlay map[string]string, outputLinkPath st return nil } -func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) { +func PatchLinker(goRoot, goVersion, cacheDir, tempDir string) (string, func(), error) { // rxVersion looks for a version like "go1.19" or "go1.20" rxVersion := regexp.MustCompile(`go\d+\.\d+`) majorGoVersion := rxVersion.FindString(goVersion) @@ -236,7 +223,7 @@ func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) { panic(fmt.Errorf("cannot retrieve linker patches: %v", err)) } - outputLinkPath, err := cachePath() + outputLinkPath, err := cachePath(cacheDir) if err != nil { return "", nil, err } diff --git a/main.go b/main.go index 3d4ae67..4328261 100644 --- a/main.go +++ b/main.go @@ -400,12 +400,15 @@ func mainErr(args []string) error { if err := os.RemoveAll(os.Getenv("GARBLE_SHARED")); err != nil { fmt.Fprintf(os.Stderr, "could not clean up GARBLE_SHARED: %v\n", err) } - fsCache, err := openCache() - if err == nil { - err = fsCache.Trim() - } - if err != nil { - fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err) + // skip the trim if we didn't even start a build + if sharedCache != nil { + fsCache, err := openCache() + if err == nil { + err = fsCache.Trim() + } + if err != nil { + fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err) + } } }() if err != nil { @@ -455,7 +458,7 @@ func mainErr(args []string) error { executablePath := args[0] if tool == "link" { - modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedTempDir) + modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedCache.CacheDir, sharedTempDir) if err != nil { return fmt.Errorf("cannot get modified linker: %v", err) } @@ -543,6 +546,20 @@ This command wraps "go %s". Below is its help: return nil, err } + // Always an absolute directory; defaults to e.g. "~/.cache/garble". + if dir := os.Getenv("GARBLE_CACHE"); dir != "" { + sharedCache.CacheDir, err = filepath.Abs(dir) + if err != nil { + return nil, err + } + } else { + parentDir, err := os.UserCacheDir() + if err != nil { + return nil, err + } + sharedCache.CacheDir = filepath.Join(parentDir, "garble") + } + binaryBuildID, err := buildidOf(sharedCache.ExecPath) if err != nil { return nil, err @@ -1304,18 +1321,9 @@ func (c *pkgCache) CopyFrom(c2 pkgCache) { } func openCache() (*cache.Cache, error) { - dir := os.Getenv("GARBLE_CACHE") // e.g. "~/.cache/garble" - if dir == "" { - parentDir, err := os.UserCacheDir() - if err != nil { - return nil, err - } - dir = filepath.Join(parentDir, "garble") - } // Use a subdirectory for the hashed build cache, to clarify what it is, // and to allow us to have other directories or files later on without mixing. - dir = filepath.Join(dir, "build") - + dir := filepath.Join(sharedCache.CacheDir, "build") if err := os.MkdirAll(dir, 0o777); err != nil { return nil, err } diff --git a/main_test.go b/main_test.go index 3161782..f1c034e 100644 --- a/main_test.go +++ b/main_test.go @@ -101,11 +101,9 @@ func TestScript(t *testing.T) { // coverage. Otherwise, the coverage info might be incomplete. env.Setenv("GOCACHE", filepath.Join(tempCacheDir, "go-cache")) env.Setenv("GARBLE_CACHE", filepath.Join(tempCacheDir, "garble-cache")) - env.Setenv("GARBLE_CACHE_DIR", filepath.Join(tempCacheDir, "garble-cache-2")) } else { // GOCACHE is initialized by gotooltest to use the host's cache. env.Setenv("GARBLE_CACHE", filepath.Join(hostCacheDir, "garble")) - env.Setenv("GARBLE_CACHE_DIR", hostCacheDir) } return nil }, diff --git a/shared.go b/shared.go index b246490..d3fabea 100644 --- a/shared.go +++ b/shared.go @@ -32,6 +32,8 @@ type sharedCacheType struct { ExecPath string // absolute path to the garble binary being used ForwardBuildFlags []string // build flags fed to the original "garble ..." command + CacheDir string // absolute path to the GARBLE_CACHE directory being used + // ListedPackages contains data obtained via 'go list -json -export -deps'. // This allows us to obtain the non-obfuscated export data of all dependencies, // useful for type checking of the packages as we obfuscate them. diff --git a/testdata/script/linker.txtar b/testdata/script/linker.txtar index 6be3e62..ebda394 100644 --- a/testdata/script/linker.txtar +++ b/testdata/script/linker.txtar @@ -1,7 +1,7 @@ # Past garble versions might not properly patch cmd/link with "git apply" # when running inside a git repository. Skip the extra check with -short. [!short] [exec:git] exec git init -q -[!short] [exec:git] env GARBLE_CACHE_DIR=${WORK}/linker-cache +[!short] [exec:git] env GARBLE_CACHE=${WORK}/garble-cache # Any build settings for the main build shouldn't affect building the linker. # If this flag makes it through when using build commands on std or cmd, @@ -20,10 +20,10 @@ exec ./main [!windows] env GOOS=windows [windows] env GOOS=linux exec garble build -[!windows] [exec:git] exists ${GARBLE_CACHE_DIR}/garble/link -[!windows] [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link.exe -[windows] [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link -[windows] [exec:git] exists ${GARBLE_CACHE_DIR}/garble/link.exe +[!windows] [exec:git] exists ${GARBLE_CACHE}/tool/link +[!windows] [exec:git] ! exists ${GARBLE_CACHE}/tool/link.exe +[windows] [exec:git] ! exists ${GARBLE_CACHE}/tool/link +[windows] [exec:git] exists ${GARBLE_CACHE}/tool/link.exe env GOOS= # Verify a build without garble.