From 34cbd1b8411086379c760381e5ff416d375f4923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 8 Jan 2022 22:21:29 +0000 Subject: [PATCH] only list missing packages when obfuscating the runtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were listing all of std, which certainly worked, but was quite slow at over 200 packages. In practice, we can only be missing up to 20-30 packages. It was a good change as it fixed a severe bug, but it also introduced a fairly noticeable slow-down. The numbers are clear; this change shaves off multiple seconds when obfuscating the runtime with a cold cache: name old time/op new time/op delta Build/NoCache-16 5.06s ± 1% 1.94s ± 1% -61.64% (p=0.008 n=5+5) name old bin-B new bin-B delta Build/NoCache-16 6.70M ± 0% 6.71M ± 0% +0.05% (p=0.008 n=5+5) name old sys-time/op new sys-time/op delta Build/NoCache-16 13.4s ± 2% 5.0s ± 2% -62.45% (p=0.008 n=5+5) name old user-time/op new user-time/op delta Build/NoCache-16 60.6s ± 1% 19.8s ± 1% -67.34% (p=0.008 n=5+5) Since we only want to call "go list" one extra time, instead of once for every package we find out we're missing, we want to know what packages we could be missing in advance. Resurrect a smarter version of the runtime-related script. Finally, remove the runtime-related.txt test script, as it has now been superseeded by the sanity checks in listPackage. That is, obfuscating the runtime package will now panic if we are missing any necessary package information. To double check that we get the runtime's linkname edge case right, make gogarble.txt use runtime/debug.WriteHeapDump, which is implemented via a direct runtime linkname. This ensures we don't lose test coverage from runtime-related.txt. --- main.go | 2 +- scripts/runtime-linknamed-nodeps.sh | 21 +++++++++++ shared.go | 42 +++++++++++++++++++-- testdata/scripts/crossbuild.txt | 4 +- testdata/scripts/gogarble.txt | 8 +++- testdata/scripts/runtime-related.txt | 55 ---------------------------- 6 files changed, 68 insertions(+), 64 deletions(-) create mode 100755 scripts/runtime-linknamed-nodeps.sh delete mode 100644 testdata/scripts/runtime-related.txt diff --git a/main.go b/main.go index 0f756a4..1886a57 100644 --- a/main.go +++ b/main.go @@ -1967,7 +1967,7 @@ func flagSetValue(flags []string, name, value string) []string { func fetchGoEnv() error { out, err := exec.Command("go", "env", "-json", - "GOPRIVATE", "GOMOD", "GOVERSION", "GOCACHE", + "GOOS", "GOPRIVATE", "GOMOD", "GOVERSION", "GOCACHE", ).CombinedOutput() if err != nil { fmt.Fprintf(os.Stderr, `Can't find Go toolchain: %v diff --git a/scripts/runtime-linknamed-nodeps.sh b/scripts/runtime-linknamed-nodeps.sh new file mode 100755 index 0000000..9ee9b64 --- /dev/null +++ b/scripts/runtime-linknamed-nodeps.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# The list of all packages the runtime source linknames to. +linked="$(sed -rn 's@//go:linkname .* ([^.]*)\.[^.]*@\1@p' $(go env GOROOT)/src/runtime/*.go | grep -vE '^main|^runtime\.' | sort -u)" + +# The list of all implied dependencies of the packages above, +# across all main GOOS targets. +implied="$(for GOOS in linux darwin windows js; do + for pkg in $linked; do + GOOS=$GOOS GOARCH=$GOARCH go list -e -deps $pkg | grep -v '^'$pkg'$' + done +done | sort -u)" + +# All packages in linked, except those implied by others already. +# This resulting list is what we need to "go list" when obfuscating the runtime, +# as they are the packages that we may be missing. +comm -23 <( + echo "$linked" +) <( + echo "$implied" +) diff --git a/shared.go b/shared.go index 9c1bb41..7beda3c 100644 --- a/shared.go +++ b/shared.go @@ -39,8 +39,11 @@ type sharedCache struct { GOGARBLE string - // From "go env", primarily. + // Filled directly from "go env". + // Remember to update the exec call when adding or removing names. GoEnv struct { + GOOS string // i.e. the GOOS build target + GOPRIVATE string GOMOD string GOVERSION string @@ -225,6 +228,8 @@ func appendListedPackages(patterns ...string) error { return nil } +var listedRuntimeLinknamed = false + // listPackage gets the listedPackage information for a certain package func listPackage(path string) (*listedPackage, error) { if path == curPkg.ImportPath { @@ -248,15 +253,44 @@ func listPackage(path string) (*listedPackage, error) { if ok { return pkg, nil } - // TODO: List fewer packages here. std is 200+ packages, - // but in reality we should only miss 20-30 packages at most. - if err := appendListedPackages("std"); err != nil { + if listedRuntimeLinknamed { + panic(fmt.Sprintf("package %q still missing after go list call", path)) + } + startTime := time.Now() + // Obtained via scripts/runtime-linknamed-nodeps.sh as of Go 1.18beta1. + runtimeLinknamed := []string{ + "crypto/x509/internal/macos", + "net", + "os/signal", + "plugin", + "runtime/debug", + "runtime/metrics", + "runtime/pprof", + "runtime/trace", + "syscall/js", + } + var missing []string + for _, linknamed := range runtimeLinknamed { + switch { + case cache.ListedPackages[linknamed] != nil: + // We already have it; skip. + case cache.GoEnv.GOOS != "js" && linknamed == "syscall/js": + // GOOS-specific package. + case cache.GoEnv.GOOS != "darwin" && linknamed == "crypto/x509/internal/macos": + // GOOS-specific package. + default: + missing = append(missing, linknamed) + } + } + if err := appendListedPackages(missing...); err != nil { panic(err) // should never happen } pkg, ok := cache.ListedPackages[path] if !ok { panic(fmt.Sprintf("runtime listed a std package we can't find: %s", path)) } + listedRuntimeLinknamed = true + debugf("listed %d missing runtime-linknamed packages in %s", len(missing), debugSince(startTime)) return pkg, nil } // Packages other than runtime can list any package, diff --git a/testdata/scripts/crossbuild.txt b/testdata/scripts/crossbuild.txt index 68ae66d..b681856 100644 --- a/testdata/scripts/crossbuild.txt +++ b/testdata/scripts/crossbuild.txt @@ -8,10 +8,8 @@ [!arm] env GOARCH=arm [arm] env GOARCH=arm64 +# A fairly average Go build, importing some std libraries. env GOGARBLE='*' - -# Link a binary importing net/http, which will catch whether or not we -# support ImportMap when linking. garble build -- go.mod -- diff --git a/testdata/scripts/gogarble.txt b/testdata/scripts/gogarble.txt index eafb655..26bee34 100644 --- a/testdata/scripts/gogarble.txt +++ b/testdata/scripts/gogarble.txt @@ -71,8 +71,14 @@ var LongString = "some long string to obfuscate" -- stdimporter/main.go -- package main -import "net/http" +import ( + "net/http" + "runtime/debug" +) func main() { http.ListenAndServe("", nil) + // debug.WriteHeapDump is particularly interesting, + // as it is implemented by runtime via a linkname. + debug.WriteHeapDump(1) } diff --git a/testdata/scripts/runtime-related.txt b/testdata/scripts/runtime-related.txt deleted file mode 100644 index 4f3f6bd..0000000 --- a/testdata/scripts/runtime-related.txt +++ /dev/null @@ -1,55 +0,0 @@ -# test that we used all necessary dependencies -[linux] exec bash runtime-related-tested.sh - -env GOPRIVATE=* -garble build - --- runtime-related.sh -- -for GOOS in linux darwin windows; do - skip="|macos" - if [[ $GOOS == "darwin" ]]; then - skip="" - fi - - GOOS=$GOOS go list -deps $(sed -rn 's@//go:linkname .* ([^.]*)\.[^.]*@\1@p' $(go env GOROOT)/src/runtime/*.go | grep -vE '^main|^runtime\.|js'$skip) runtime || exit 1 -done | sort -u - --- runtime-related-tested.sh -- -# get all runtime-related deps -related=$(bash runtime-related.sh) || exit 1 - -# get all tested deps -tested=$(for GOOS in linux darwin windows; do GOOS=$GOOS go list -deps || exit 1; done | sort -u) - -# remove all tested deps from the runtime-related deps -output=$(echo "$related" | grep -Fvx -e "$tested") - -# output should be empty if all runtime-related deps are tested -[[ -z "$output" ]] || (echo "$output" && exit 1) - --- go.mod -- -module test/main - -go 1.17 - --- main.go -- -package main - -import ( - "net/http/pprof" - "os/signal" - "plugin" - "runtime/debug" - "runtime/metrics" - "text/tabwriter" -) - -// This program imports all runtime-related dependencies (proven by runtime-related-tested.sh) -func main() { - _ = tabwriter.AlignRight - signal.Ignore() - _ = plugin.Plugin{} - _ = pprof.Handler("") - _ = debug.GCStats{} - metrics.All() -}