only list missing packages when obfuscating the runtime

We were listing all of std, which certainly worked,
but was quite slow at over 200 packages.
In practice, we can only be missing up to 20-30 packages.
It was a good change as it fixed a severe bug,
but it also introduced a fairly noticeable slow-down.

The numbers are clear; this change shaves off multiple seconds when
obfuscating the runtime with a cold cache:

	name              old time/op       new time/op       delta
	Build/NoCache-16        5.06s ± 1%        1.94s ± 1%  -61.64%  (p=0.008 n=5+5)

	name              old bin-B         new bin-B         delta
	Build/NoCache-16        6.70M ± 0%        6.71M ± 0%   +0.05%  (p=0.008 n=5+5)

	name              old sys-time/op   new sys-time/op   delta
	Build/NoCache-16        13.4s ± 2%         5.0s ± 2%  -62.45%  (p=0.008 n=5+5)

	name              old user-time/op  new user-time/op  delta
	Build/NoCache-16        60.6s ± 1%        19.8s ± 1%  -67.34%  (p=0.008 n=5+5)

Since we only want to call "go list" one extra time,
instead of once for every package we find out we're missing,
we want to know what packages we could be missing in advance.
Resurrect a smarter version of the runtime-related script.

Finally, remove the runtime-related.txt test script,
as it has now been superseeded by the sanity checks in listPackage.
That is, obfuscating the runtime package will now panic if we are
missing any necessary package information.

To double check that we get the runtime's linkname edge case right,
make gogarble.txt use runtime/debug.WriteHeapDump,
which is implemented via a direct runtime linkname.
This ensures we don't lose test coverage from runtime-related.txt.

main.go

@@ -1967,7 +1967,7 @@ func flagSetValue(flags []string, name, value string) []string {
 
 func fetchGoEnv() error {
 	out, err := exec.Command("go", "env", "-json",
-		"GOPRIVATE", "GOMOD", "GOVERSION", "GOCACHE",
+		"GOOS", "GOPRIVATE", "GOMOD", "GOVERSION", "GOCACHE",
 	).CombinedOutput()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, `Can't find Go toolchain: %v

scripts/runtime-linknamed-nodeps.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# The list of all packages the runtime source linknames to.
+linked="$(sed -rn 's@//go:linkname .* ([^.]*)\.[^.]*@\1@p' $(go env GOROOT)/src/runtime/*.go | grep -vE '^main|^runtime\.' | sort -u)"
+
+# The list of all implied dependencies of the packages above,
+# across all main GOOS targets.
+implied="$(for GOOS in linux darwin windows js; do
+	for pkg in $linked; do
+		GOOS=$GOOS GOARCH=$GOARCH go list -e -deps $pkg | grep -v '^'$pkg'$'
+	done
+done | sort -u)"
+
+# All packages in linked, except those implied by others already.
+# This resulting list is what we need to "go list" when obfuscating the runtime,
+# as they are the packages that we may be missing.
+comm -23 <(
+	echo "$linked"
+) <(
+	echo "$implied"
+)

shared.go

@@ -39,8 +39,11 @@ type sharedCache struct {
 
 	GOGARBLE string
 
-	// From "go env", primarily.
+	// Filled directly from "go env".
+	// Remember to update the exec call when adding or removing names.
 	GoEnv struct {
+		GOOS string // i.e. the GOOS build target
+
 		GOPRIVATE string
 		GOMOD     string
 		GOVERSION string
@@ -225,6 +228,8 @@ func appendListedPackages(patterns ...string) error {
 	return nil
 }
 
+var listedRuntimeLinknamed = false
+
 // listPackage gets the listedPackage information for a certain package
 func listPackage(path string) (*listedPackage, error) {
 	if path == curPkg.ImportPath {
@@ -248,15 +253,44 @@ func listPackage(path string) (*listedPackage, error) {
 		if ok {
 			return pkg, nil
 		}
-		// TODO: List fewer packages here. std is 200+ packages,
-		// but in reality we should only miss 20-30 packages at most.
-		if err := appendListedPackages("std"); err != nil {
+		if listedRuntimeLinknamed {
+			panic(fmt.Sprintf("package %q still missing after go list call", path))
+		}
+		startTime := time.Now()
+		// Obtained via scripts/runtime-linknamed-nodeps.sh as of Go 1.18beta1.
+		runtimeLinknamed := []string{
+			"crypto/x509/internal/macos",
+			"net",
+			"os/signal",
+			"plugin",
+			"runtime/debug",
+			"runtime/metrics",
+			"runtime/pprof",
+			"runtime/trace",
+			"syscall/js",
+		}
+		var missing []string
+		for _, linknamed := range runtimeLinknamed {
+			switch {
+			case cache.ListedPackages[linknamed] != nil:
+				// We already have it; skip.
+			case cache.GoEnv.GOOS != "js" && linknamed == "syscall/js":
+				// GOOS-specific package.
+			case cache.GoEnv.GOOS != "darwin" && linknamed == "crypto/x509/internal/macos":
+				// GOOS-specific package.
+			default:
+				missing = append(missing, linknamed)
+			}
+		}
+		if err := appendListedPackages(missing...); err != nil {
 			panic(err) // should never happen
 		}
 		pkg, ok := cache.ListedPackages[path]
 		if !ok {
 			panic(fmt.Sprintf("runtime listed a std package we can't find: %s", path))
 		}
+		listedRuntimeLinknamed = true
+		debugf("listed %d missing runtime-linknamed packages in %s", len(missing), debugSince(startTime))
 		return pkg, nil
 	}
 	// Packages other than runtime can list any package,

testdata/scripts/crossbuild.txt

@@ -8,10 +8,8 @@
 [!arm] env GOARCH=arm
 [arm] env GOARCH=arm64
 
+# A fairly average Go build, importing some std libraries.
 env GOGARBLE='*'
-
-# Link a binary importing net/http, which will catch whether or not we
-# support ImportMap when linking.
 garble build
 
 -- go.mod --

testdata/scripts/gogarble.txt

@@ -71,8 +71,14 @@ var LongString = "some long string to obfuscate"
 -- stdimporter/main.go --
 package main
 
-import "net/http"
+import (
+	"net/http"
+	"runtime/debug"
+)
 
 func main() {
 	http.ListenAndServe("", nil)
+	// debug.WriteHeapDump is particularly interesting,
+	// as it is implemented by runtime via a linkname.
+	debug.WriteHeapDump(1)
 }

testdata/scripts/runtime-related.txt

@@ -1,55 +0,0 @@
-# test that we used all necessary dependencies
-[linux] exec bash runtime-related-tested.sh
-
-env GOPRIVATE=*
-garble build
-
--- runtime-related.sh --
-for GOOS in linux darwin windows; do
-	skip="|macos"
-	if [[ $GOOS == "darwin" ]]; then
-		skip=""
-	fi
-
-	GOOS=$GOOS go list -deps $(sed -rn 's@//go:linkname .* ([^.]*)\.[^.]*@\1@p' $(go env GOROOT)/src/runtime/*.go | grep -vE '^main|^runtime\.|js'$skip) runtime || exit 1
-done | sort -u
-
--- runtime-related-tested.sh --
-# get all runtime-related deps
-related=$(bash runtime-related.sh) || exit 1
-
-# get all tested deps
-tested=$(for GOOS in linux darwin windows; do GOOS=$GOOS go list -deps || exit 1; done | sort -u)
-
-# remove all tested deps from the runtime-related deps 
-output=$(echo "$related" | grep -Fvx -e "$tested")
-
-# output should be empty if all runtime-related deps are tested
-[[ -z "$output" ]] || (echo "$output" && exit 1)
-
--- go.mod --
-module test/main
-
-go 1.17
-
--- main.go --
-package main
-
-import (
-	"net/http/pprof"
-	"os/signal"
-	"plugin"
-	"runtime/debug"
-	"runtime/metrics"
-	"text/tabwriter"
-)
-
-// This program imports all runtime-related dependencies (proven by runtime-related-tested.sh)
-func main() {
-	_ = tabwriter.AlignRight
-	signal.Ignore()
-	_ = plugin.Plugin{}
-	_ = pprof.Handler("")
-	_ = debug.GCStats{}
-	metrics.All()
-}