diff --git a/README.md b/README.md
index cd0867f..8059e8d 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,10 @@ Note that the first call to `garble build` may be comparatively slow,
 as it has to obfuscate each package for the first time. This is akin to clearing
 `GOCACHE` with `go clean -cache` and running a `go build` from scratch.
 
+Garble also makes use of its own cache to reuse work, akin to Go's `GOCACHE`.
+It defaults to a directory under your user's cache directory,
+such as `~/.cache/garble`, and can be placed elsewhere by setting `GARBLE_CACHE`.
+
 ### Determinism and seeds
 
 Just like Go, garble builds are deterministic and reproducible in nature.
diff --git a/internal/linker/linker.go b/internal/linker/linker.go
index dce2764..f3bb932 100644
--- a/internal/linker/linker.go
+++ b/internal/linker/linker.go
@@ -27,10 +27,8 @@ const (
 	TinyEnv        = "GARBLE_LINK_TINY"
 	EntryOffKeyEnv = "GARBLE_LINK_ENTRYOFF_KEY"
 
-	cacheDirName   = "garble"
-	versionExt     = ".version"
-	garbleCacheDir = "GARBLE_CACHE_DIR"
-	baseSrcSubdir  = "src"
+	versionExt    = ".version"
+	baseSrcSubdir = "src"
 )
 
 //go:embed patches/*/*.patch
@@ -137,25 +135,14 @@ func applyPatches(srcDir, workingDir string, modFiles map[string]bool, patches [
 	return mod, nil
 }
 
-// TODO: put linker binaries into fsCache in the main package
-
-func cachePath() (string, error) {
-	var cacheDir string
-	if val, ok := os.LookupEnv(garbleCacheDir); ok {
-		cacheDir = val
-	} else {
-		userCacheDir, err := os.UserCacheDir()
-		if err != nil {
-			panic(fmt.Errorf("cannot retreive user cache directory: %v", err))
-		}
-		cacheDir = userCacheDir
-	}
-
-	cacheDir = filepath.Join(cacheDir, cacheDirName)
+func cachePath(cacheDir string) (string, error) {
+	// Use a subdirectory to clarify what we're using it for.
+	// Name it "tool", like Go's pkg/tool, as we might want to rebuild
+	// other Go toolchain programs like the compiler or assembler in the future.
+	cacheDir = filepath.Join(cacheDir, "tool")
 	if err := os.MkdirAll(cacheDir, 0o777); err != nil {
 		return "", err
 	}
-
 	goExe := ""
 	if runtime.GOOS == "windows" {
 		goExe = ".exe"
@@ -226,7 +213,7 @@ func buildLinker(workingDir string, overlay map[string]string, outputLinkPath st
 	return nil
 }
 
-func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) {
+func PatchLinker(goRoot, goVersion, cacheDir, tempDir string) (string, func(), error) {
 	// rxVersion looks for a version like "go1.19" or "go1.20"
 	rxVersion := regexp.MustCompile(`go\d+\.\d+`)
 	majorGoVersion := rxVersion.FindString(goVersion)
@@ -236,7 +223,7 @@ func PatchLinker(goRoot, goVersion, tempDir string) (string, func(), error) {
 		panic(fmt.Errorf("cannot retrieve linker patches: %v", err))
 	}
 
-	outputLinkPath, err := cachePath()
+	outputLinkPath, err := cachePath(cacheDir)
 	if err != nil {
 		return "", nil, err
 	}
diff --git a/main.go b/main.go
index 3d4ae67..4328261 100644
--- a/main.go
+++ b/main.go
@@ -400,12 +400,15 @@ func mainErr(args []string) error {
 			if err := os.RemoveAll(os.Getenv("GARBLE_SHARED")); err != nil {
 				fmt.Fprintf(os.Stderr, "could not clean up GARBLE_SHARED: %v\n", err)
 			}
-			fsCache, err := openCache()
-			if err == nil {
-				err = fsCache.Trim()
-			}
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err)
+			// skip the trim if we didn't even start a build
+			if sharedCache != nil {
+				fsCache, err := openCache()
+				if err == nil {
+					err = fsCache.Trim()
+				}
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "could not trim GARBLE_CACHE: %v\n", err)
+				}
 			}
 		}()
 		if err != nil {
@@ -455,7 +458,7 @@ func mainErr(args []string) error {
 
 		executablePath := args[0]
 		if tool == "link" {
-			modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedTempDir)
+			modifiedLinkPath, unlock, err := linker.PatchLinker(sharedCache.GoEnv.GOROOT, sharedCache.GoEnv.GOVERSION, sharedCache.CacheDir, sharedTempDir)
 			if err != nil {
 				return fmt.Errorf("cannot get modified linker: %v", err)
 			}
@@ -543,6 +546,20 @@ This command wraps "go %s". Below is its help:
 		return nil, err
 	}
 
+	// Always an absolute directory; defaults to e.g. "~/.cache/garble".
+	if dir := os.Getenv("GARBLE_CACHE"); dir != "" {
+		sharedCache.CacheDir, err = filepath.Abs(dir)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		parentDir, err := os.UserCacheDir()
+		if err != nil {
+			return nil, err
+		}
+		sharedCache.CacheDir = filepath.Join(parentDir, "garble")
+	}
+
 	binaryBuildID, err := buildidOf(sharedCache.ExecPath)
 	if err != nil {
 		return nil, err
@@ -1304,18 +1321,9 @@ func (c *pkgCache) CopyFrom(c2 pkgCache) {
 }
 
 func openCache() (*cache.Cache, error) {
-	dir := os.Getenv("GARBLE_CACHE") // e.g. "~/.cache/garble"
-	if dir == "" {
-		parentDir, err := os.UserCacheDir()
-		if err != nil {
-			return nil, err
-		}
-		dir = filepath.Join(parentDir, "garble")
-	}
 	// Use a subdirectory for the hashed build cache, to clarify what it is,
 	// and to allow us to have other directories or files later on without mixing.
-	dir = filepath.Join(dir, "build")
-
+	dir := filepath.Join(sharedCache.CacheDir, "build")
 	if err := os.MkdirAll(dir, 0o777); err != nil {
 		return nil, err
 	}
diff --git a/main_test.go b/main_test.go
index 3161782..f1c034e 100644
--- a/main_test.go
+++ b/main_test.go
@@ -101,11 +101,9 @@ func TestScript(t *testing.T) {
 				// coverage. Otherwise, the coverage info might be incomplete.
 				env.Setenv("GOCACHE", filepath.Join(tempCacheDir, "go-cache"))
 				env.Setenv("GARBLE_CACHE", filepath.Join(tempCacheDir, "garble-cache"))
-				env.Setenv("GARBLE_CACHE_DIR", filepath.Join(tempCacheDir, "garble-cache-2"))
 			} else {
 				// GOCACHE is initialized by gotooltest to use the host's cache.
 				env.Setenv("GARBLE_CACHE", filepath.Join(hostCacheDir, "garble"))
-				env.Setenv("GARBLE_CACHE_DIR", hostCacheDir)
 			}
 			return nil
 		},
diff --git a/shared.go b/shared.go
index b246490..d3fabea 100644
--- a/shared.go
+++ b/shared.go
@@ -32,6 +32,8 @@ type sharedCacheType struct {
 	ExecPath          string   // absolute path to the garble binary being used
 	ForwardBuildFlags []string // build flags fed to the original "garble ..." command
 
+	CacheDir string // absolute path to the GARBLE_CACHE directory being used
+
 	// ListedPackages contains data obtained via 'go list -json -export -deps'.
 	// This allows us to obtain the non-obfuscated export data of all dependencies,
 	// useful for type checking of the packages as we obfuscate them.
diff --git a/testdata/script/linker.txtar b/testdata/script/linker.txtar
index 6be3e62..ebda394 100644
--- a/testdata/script/linker.txtar
+++ b/testdata/script/linker.txtar
@@ -1,7 +1,7 @@
 # Past garble versions might not properly patch cmd/link with "git apply"
 # when running inside a git repository. Skip the extra check with -short.
 [!short] [exec:git] exec git init -q
-[!short] [exec:git] env GARBLE_CACHE_DIR=${WORK}/linker-cache
+[!short] [exec:git] env GARBLE_CACHE=${WORK}/garble-cache
 
 # Any build settings for the main build shouldn't affect building the linker.
 # If this flag makes it through when using build commands on std or cmd,
@@ -20,10 +20,10 @@ exec ./main
 [!windows] env GOOS=windows
 [windows] env GOOS=linux
 exec garble build
-[!windows] [exec:git]   exists ${GARBLE_CACHE_DIR}/garble/link
-[!windows] [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link.exe
-[windows]  [exec:git] ! exists ${GARBLE_CACHE_DIR}/garble/link
-[windows]  [exec:git]   exists ${GARBLE_CACHE_DIR}/garble/link.exe
+[!windows] [exec:git]   exists ${GARBLE_CACHE}/tool/link
+[!windows] [exec:git] ! exists ${GARBLE_CACHE}/tool/link.exe
+[windows]  [exec:git] ! exists ${GARBLE_CACHE}/tool/link
+[windows]  [exec:git]   exists ${GARBLE_CACHE}/tool/link.exe
 env GOOS=
 
 # Verify a build without garble.