@ -4,80 +4,121 @@
package main package main
import ( import (
_ "embed"
"flag"
"fmt"
"os"
"os"
"os/exec"
"os/exec"
"path/filepath"
"path/filepath"
"regexp"
"runtime"
"runtime"
"strings"
"testing"
"testing"
"time"
qt "github.com/frankban/quicktest"
) )
// BenchmarkBuild is a parallel benchmark for 'garble build' on a fairly simple
//go:embed testdata/bench/main.go
var benchSourceMain [ ] byte
var (
rxBuiltRuntime = regexp . MustCompile ( ` (?m)^runtime$ ` )
rxBuiltMain = regexp . MustCompile ( ` (?m)^test/main$ ` )
)
// BenchmarkBuild is a benchmark for 'garble build' on a fairly simple
// main package with a handful of standard library depedencies.
// main package with a handful of standard library depedencies.
//
//
// We use a real garble binary and exec it, to simulate what the real user would
// We use a real garble binary and exec it, to simulate what the real user would
// run. The real obfuscation and compilation will happen in sub-processes
// run. The real obfuscation and compilation will happen in sub-processes
// anyway, so skipping one exec layer doesn't help us in any way.
// anyway, so skipping one exec layer doesn't help us in any way.
//
//
// The benchmark isn't parallel, because in practice users build once at a time,
// and each build already spawns concurrent processes and goroutines to do work.
//
// At the moment, each iteration takes 1-2s on a laptop, so we can't make the
// At the moment, each iteration takes 1-2s on a laptop, so we can't make the
// benchmark include any more features unless we make it significantly faster.
// benchmark include any more features unless we make it significantly faster.
func BenchmarkBuild ( b * testing . B ) { func BenchmarkBuild ( b * testing . B ) {
// As of Go 1.17, using -benchtime=Nx with N larger than 1 results in two
// calls to BenchmarkBuild, with the first having b.N==1 to discover
// sub-benchmarks. Unfortunately, we do a significant amount of work both
// during setup and during that first iteration, which is pointless.
// To avoid that, detect the scenario in a hacky way, and return early.
// See https://github.com/golang/go/issues/32051.
benchtime := flag . Lookup ( "test.benchtime" ) . Value . String ( )
if b . N == 1 && strings . HasSuffix ( benchtime , "x" ) && benchtime != "1x" {
return
}
garbleBin := filepath . Join ( b . TempDir ( ) , "garble" )
garbleBin := filepath . Join ( b . TempDir ( ) , "garble" )
if runtime . GOOS == "windows" {
if runtime . GOOS == "windows" {
garbleBin += ".exe"
garbleBin += ".exe"
}
}
err := exec . Command ( "go" , "build" , "-o=" + garbleBin ) . Run ( )
qt . Assert ( b , err , qt . IsNil )
// We collect extra metrics.
var userTime , systemTime int64
var cachedTime int64
if err := exec . Command ( "go" , "build" , "-o=" + garbleBin ) . Run ( ) ; err != nil {
outputBin := filepath . Join ( b . TempDir ( ) , "output" )
b . Fatalf ( "building garble: %v" , err )
sourceDir := filepath . Join ( b . TempDir ( ) , "src" )
err = os . Mkdir ( sourceDir , 0 o777 )
qt . Assert ( b , err , qt . IsNil )
writeSourceFile := func ( name string , content [ ] byte ) {
err := os . WriteFile ( filepath . Join ( sourceDir , name ) , content , 0 o666 )
qt . Assert ( b , err , qt . IsNil )
}
}
writeSourceFile ( "go.mod" , [ ] byte ( "module test/main" ) )
writeSourceFile ( "main.go" , benchSourceMain )
b . ResetTimer ( )
for i := 0 ; i < b . N ; i ++ {
// First we do a fresh build, using a new GOCACHE.
// and the second does an incremental rebuild reusing the cache.
gocache , err := os . MkdirTemp ( b . TempDir ( ) , "gocache-*" )
qt . Assert ( b , err , qt . IsNil )
env := append ( os . Environ ( ) , "GOGARBLE=*" , "GOCACHE=" + gocache )
args := [ ] string { "build" , "-v" , "-o=" + outputBin , sourceDir }
cmd := exec . Command ( garbleBin , args ... )
cmd . Env = env
cmd . Dir = sourceDir
out , err := cmd . CombinedOutput ( )
qt . Assert ( b , err , qt . IsNil , qt . Commentf ( "output: %s" , out ) )
// Ensure that we built all packages, as expected.
qt . Assert ( b , rxBuiltRuntime . Match ( out ) , qt . IsTrue )
qt . Assert ( b , rxBuiltMain . Match ( out ) , qt . IsTrue )
for _ , name := range [ ... ] string { "Cache" , "NoCache" } {
// The cached rebuild will reuse all dependencies,
b . Run ( name , func ( b * testing . B ) {
// but rebuild the main package itself.
buildArgs := [ ] string { "build" , "-o=" + b . TempDir ( ) }
cachedStart := time . Now ( )
switch name {
writeSourceFile ( "rebuild.go" , [ ] byte ( fmt . Sprintf ( "package main\nvar v%d int" , i ) ) )
case "Cache" :
buildArgs = append ( buildArgs , "./testdata/bench-cache" )
cmd = exec . Command ( garbleBin , args ... )
cmd . Env = env
// Ensure the build cache is warm,
cmd . Dir = sourceDir
// for the sake of consistent results.
cmd := exec . Command ( garbleBin , buildArgs ... )
out , err = cmd . CombinedOutput ( )
if out , err := cmd . CombinedOutput ( ) ; err != nil {
qt . Assert ( b , err , qt . IsNil , qt . Commentf ( "output: %s" , out ) )
b . Fatalf ( "%v: %s" , err , out )
// Ensure that we only rebuilt the main package, as expected.
}
qt . Assert ( b , rxBuiltRuntime . Match ( out ) , qt . IsFalse )
case "NoCache" :
qt . Assert ( b , rxBuiltMain . Match ( out ) , qt . IsTrue )
buildArgs = append ( buildArgs , "./testdata/bench-nocache" )
default :
userTime += int64 ( cmd . ProcessState . UserTime ( ) )
b . Fatalf ( "unknown name: %q" , name )
systemTime += int64 ( cmd . ProcessState . SystemTime ( ) )
}
cachedTime += time . Since ( cachedStart ) . Nanoseconds ( )
}
// We collect extra metrics.
b . ReportMetric ( float64 ( cachedTime ) / float64 ( b . N ) , "cached-ns/op" )
var userTime , systemTime int64
b . ReportMetric ( float64 ( userTime ) / float64 ( b . N ) , "user-ns/op" )
b . ReportMetric ( float64 ( systemTime ) / float64 ( b . N ) , "sys-ns/op" )
b . ResetTimer ( )
info , err := os . Stat ( outputBin )
b . RunParallel ( func ( pb * testing . PB ) {
if err != nil {
for pb . Next ( ) {
b . Fatal ( err )
cmd := exec . Command ( garbleBin , buildArgs ... )
if name == "NoCache" {
gocache , err := os . MkdirTemp ( b . TempDir ( ) , "gocache-*" )
if err != nil {
b . Fatal ( err )
}
cmd . Env = append ( os . Environ ( ) , "GOCACHE=" + gocache )
}
if out , err := cmd . CombinedOutput ( ) ; err != nil {
b . Fatalf ( "%v: %s" , err , out )
}
userTime += int64 ( cmd . ProcessState . UserTime ( ) )
systemTime += int64 ( cmd . ProcessState . SystemTime ( ) )
}
} )
b . ReportMetric ( float64 ( userTime ) / float64 ( b . N ) , "user-ns/op" )
b . ReportMetric ( float64 ( systemTime ) / float64 ( b . N ) , "sys-ns/op" )
info , err := os . Stat ( garbleBin )
if err != nil {
b . Fatal ( err )
}
b . ReportMetric ( float64 ( info . Size ( ) ) , "bin-B" )
} )
}
}
b . ReportMetric ( float64 ( info . Size ( ) ) , "bin-B" )
} }
Daniel Martí
3 years ago