rework the build benchmarks

First, stop writing binaries into the current directory, which pollutes
the git clone.

Second, split the benchmark into two. The old benchmark always used the
build cache after the first iteration, meaning that we weren't really
measuring the cost of cold fresh builds.

The new benchmarks show a build with an always-warm cache, and one
without any cache.

Note that NoCache with the main package importing "fmt" took about 4s
wall time, which makes benchmarking too slow. For that reason, the new
bench-nocache program has no std dependencies other than runtime, which
already pulls in half a dozen dependencies we recompile at every
iteration. This reduces the wall time to 2s, which is bearable.

On the other hand, Cache is already fast, so we add a second and
slightly heavier dependency, net/http. The build still takes under 300ms
of wall time. This also helps the Cache benchmark imitate larger rebuilds
with a warm cache.

Longer term, both benchmarks will be useful, because we want both
scenarios to be as efficient as possible.

	name             time/op
	Build/Cache-8      161ms ± 1%
	Build/NoCache-8    1.21s ± 1%

	name             bin-B
	Build/Cache-8      6.35M ± 0%
	Build/NoCache-8    6.35M ± 0%

	name             sys-time/op
	Build/Cache-8      218ms ± 7%
	Build/NoCache-8    522ms ± 4%

	name             user-time/op
	Build/Cache-8      825ms ± 1%
	Build/NoCache-8    8.17s ± 1%

bench_test.go

@@ -8,7 +8,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"sync/atomic"
 	"testing"
 )
 
@@ -31,27 +30,54 @@ func BenchmarkBuild(b *testing.B) {
 		b.Fatalf("building garble: %v", err)
 	}
 
-	// We collect extra metrics.
-	var n, userTime, systemTime int64
+	for _, name := range [...]string{"Cache", "NoCache"} {
+		b.Run(name, func(b *testing.B) {
+			buildArgs := []string{"build", "-o=" + b.TempDir()}
+			switch name {
+			case "Cache":
+				buildArgs = append(buildArgs, "./testdata/bench-cache")
 
-	b.ResetTimer()
-	b.RunParallel(func(pb *testing.PB) {
-		for pb.Next() {
-			cmd := exec.Command(garbleBin, "build", "./testdata/bench")
-			if out, err := cmd.CombinedOutput(); err != nil {
-				b.Fatalf("%v: %s", err, out)
+				// Ensure the build cache is warm,
+				// for the sake of consistent results.
+				cmd := exec.Command(garbleBin, buildArgs...)
+				if out, err := cmd.CombinedOutput(); err != nil {
+					b.Fatalf("%v: %s", err, out)
+				}
+			case "NoCache":
+				buildArgs = append(buildArgs, "./testdata/bench-nocache")
+			default:
+				b.Fatalf("unknown name: %q", name)
 			}
 
-			atomic.AddInt64(&n, 1)
-			atomic.AddInt64(&userTime, int64(cmd.ProcessState.UserTime()))
-			atomic.AddInt64(&systemTime, int64(cmd.ProcessState.SystemTime()))
-		}
-	})
-	b.ReportMetric(float64(userTime)/float64(n), "user-ns/op")
-	b.ReportMetric(float64(systemTime)/float64(n), "sys-ns/op")
-	info, err := os.Stat(garbleBin)
-	if err != nil {
-		b.Fatal(err)
+			// We collect extra metrics.
+			var userTime, systemTime int64
+
+			b.ResetTimer()
+			b.RunParallel(func(pb *testing.PB) {
+				for pb.Next() {
+					cmd := exec.Command(garbleBin, buildArgs...)
+					if name == "NoCache" {
+						gocache, err := os.MkdirTemp(b.TempDir(), "gocache-*")
+						if err != nil {
+							b.Fatal(err)
+						}
+						cmd.Env = append(os.Environ(), "GOCACHE="+gocache)
+					}
+					if out, err := cmd.CombinedOutput(); err != nil {
+						b.Fatalf("%v: %s", err, out)
+					}
+
+					userTime += int64(cmd.ProcessState.UserTime())
+					systemTime += int64(cmd.ProcessState.SystemTime())
+				}
+			})
+			b.ReportMetric(float64(userTime)/float64(b.N), "user-ns/op")
+			b.ReportMetric(float64(systemTime)/float64(b.N), "sys-ns/op")
+			info, err := os.Stat(garbleBin)
+			if err != nil {
+				b.Fatal(err)
+			}
+			b.ReportMetric(float64(info.Size()), "bin-B")
+		})
 	}
-	b.ReportMetric(float64(info.Size()), "bin-B")
 }

testdata/bench-cache/main.go

@@ -1,9 +1,15 @@
 // Copyright (c) 2020, The Garble Authors.
 // See LICENSE for licensing information.
 
+// A simple main package with some names to obfuscate.
+// With relatively heavy dependencies, as benchmark iterations use the build cache.
+
 package main
 
-import "fmt"
+import (
+	"fmt"
+	"net/http"
+)
 
 var globalVar = "global value"
 
@@ -12,4 +18,5 @@ func globalFunc() { fmt.Println("global func body") }
 func main() {
 	fmt.Println(globalVar)
 	globalFunc()
+	http.ListenAndServe("", nil)
 }

testdata/bench-nocache/main.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2020, The Garble Authors.
+// See LICENSE for licensing information.
+
+// A simple main package with some names to obfuscate.
+// No dependencies, since each benchmark iteration will rebuild all deps.
+
+package main
+
+var globalVar = "global value"
+
+func globalFunc() { println("global func body") }
+
+func main() {
+	println(globalVar)
+	globalFunc()
+}