From c1439947f9d25ef776154e01a390d3df55297d75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 31 Aug 2024 00:25:10 +0100 Subject: [PATCH] generate Go tables with a list of GOTOOLCHAIN versions This teaches the program how to collect information from multiple Go versions and join it together. For this to work, it needs to select the Go versions itself, which is now possible via GOTOOLCHAIN. The merging of data is fairly simple; we join the results from all versions, and we remove duplicates from older Go versions. Start producing output with the Go version noted on every data point, so that we can easily scan what each Go version is contributing. --- go_std_tables.go | 336 +++++++++++++++++------------------ scripts/gen_go_std_tables.go | 167 +++++++++++------ 2 files changed, 278 insertions(+), 225 deletions(-) diff --git a/go_std_tables.go b/go_std_tables.go index 75b549f..11aa175 100644 --- a/go_std_tables.go +++ b/go_std_tables.go @@ -1,53 +1,53 @@ // Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT. -// Generated from Go version go1.22.6. +// Generated from Go versions [go1.22.6]. package main var runtimeAndDeps = map[string]bool{ - "internal/abi": true, - "internal/bytealg": true, - "internal/chacha8rand": true, - "internal/coverage/rtcov": true, - "internal/cpu": true, - "internal/goarch": true, - "internal/godebugs": true, - "internal/goexperiment": true, - "internal/goos": true, - "runtime": true, - "runtime/internal/atomic": true, - "runtime/internal/math": true, - "runtime/internal/sys": true, - "runtime/internal/syscall": true, - "unsafe": true, + "internal/abi": true, // go1.22.6 + "internal/bytealg": true, // go1.22.6 + "internal/chacha8rand": true, // go1.22.6 + "internal/coverage/rtcov": true, // go1.22.6 + "internal/cpu": true, // go1.22.6 + "internal/goarch": true, // go1.22.6 + "internal/godebugs": true, // go1.22.6 + "internal/goexperiment": true, // go1.22.6 + "internal/goos": true, // go1.22.6 + "runtime": true, // go1.22.6 + "runtime/internal/atomic": true, // go1.22.6 + "runtime/internal/math": true, // go1.22.6 + "runtime/internal/sys": true, // go1.22.6 + "runtime/internal/syscall": true, // go1.22.6 + "unsafe": true, // go1.22.6 } var runtimeLinknamed = []string{ - "arena", - "crypto/internal/boring", - "crypto/internal/boring/bcache", - "crypto/internal/boring/fipstls", - "crypto/x509/internal/macos", - "internal/godebug", - "internal/poll", - "internal/reflectlite", - "internal/syscall/unix", - "internal/syscall/windows", - "maps", - "os", - "os/signal", - "plugin", - "reflect", - "runtime/coverage", - "runtime/debug", - "runtime/metrics", - "runtime/pprof", - "runtime/trace", - "sync", - "sync/atomic", - "syscall", - "syscall/js", - "time", + "arena", // go1.22.6 + "crypto/internal/boring", // go1.22.6 + "crypto/internal/boring/bcache", // go1.22.6 + "crypto/internal/boring/fipstls", // go1.22.6 + "crypto/x509/internal/macos", // go1.22.6 + "internal/godebug", // go1.22.6 + "internal/poll", // go1.22.6 + "internal/reflectlite", // go1.22.6 + "internal/syscall/unix", // go1.22.6 + "internal/syscall/windows", // go1.22.6 + "maps", // go1.22.6 + "os", // go1.22.6 + "os/signal", // go1.22.6 + "plugin", // go1.22.6 + "reflect", // go1.22.6 + "runtime/coverage", // go1.22.6 + "runtime/debug", // go1.22.6 + "runtime/metrics", // go1.22.6 + "runtime/pprof", // go1.22.6 + "runtime/trace", // go1.22.6 + "sync", // go1.22.6 + "sync/atomic", // go1.22.6 + "syscall", // go1.22.6 + "syscall/js", // go1.22.6 + "time", // go1.22.6 // The net package linknames to the runtime, not the other way around. // TODO: support this automatically via our script. "net", @@ -55,149 +55,149 @@ var runtimeLinknamed = []string{ var compilerIntrinsics = map[string]map[string]bool{ "math": { - "Abs": true, - "Ceil": true, - "Copysign": true, - "FMA": true, - "Floor": true, - "Round": true, - "RoundToEven": true, - "Trunc": true, - "sqrt": true, + "Abs": true, // go1.22.6 + "Ceil": true, // go1.22.6 + "Copysign": true, // go1.22.6 + "FMA": true, // go1.22.6 + "Floor": true, // go1.22.6 + "Round": true, // go1.22.6 + "RoundToEven": true, // go1.22.6 + "Trunc": true, // go1.22.6 + "sqrt": true, // go1.22.6 }, "math/big": { - "mulWW": true, + "mulWW": true, // go1.22.6 }, "math/bits": { - "Add": true, - "Add64": true, - "Div": true, - "Div64": true, - "Len": true, - "Len16": true, - "Len32": true, - "Len64": true, - "Len8": true, - "Mul": true, - "Mul64": true, - "OnesCount": true, - "OnesCount16": true, - "OnesCount32": true, - "OnesCount64": true, - "OnesCount8": true, - "Reverse": true, - "Reverse16": true, - "Reverse32": true, - "Reverse64": true, - "Reverse8": true, - "ReverseBytes16": true, - "ReverseBytes32": true, - "ReverseBytes64": true, - "RotateLeft": true, - "RotateLeft16": true, - "RotateLeft32": true, - "RotateLeft64": true, - "RotateLeft8": true, - "Sub": true, - "Sub64": true, - "TrailingZeros16": true, - "TrailingZeros32": true, - "TrailingZeros64": true, - "TrailingZeros8": true, + "Add": true, // go1.22.6 + "Add64": true, // go1.22.6 + "Div": true, // go1.22.6 + "Div64": true, // go1.22.6 + "Len": true, // go1.22.6 + "Len16": true, // go1.22.6 + "Len32": true, // go1.22.6 + "Len64": true, // go1.22.6 + "Len8": true, // go1.22.6 + "Mul": true, // go1.22.6 + "Mul64": true, // go1.22.6 + "OnesCount": true, // go1.22.6 + "OnesCount16": true, // go1.22.6 + "OnesCount32": true, // go1.22.6 + "OnesCount64": true, // go1.22.6 + "OnesCount8": true, // go1.22.6 + "Reverse": true, // go1.22.6 + "Reverse16": true, // go1.22.6 + "Reverse32": true, // go1.22.6 + "Reverse64": true, // go1.22.6 + "Reverse8": true, // go1.22.6 + "ReverseBytes16": true, // go1.22.6 + "ReverseBytes32": true, // go1.22.6 + "ReverseBytes64": true, // go1.22.6 + "RotateLeft": true, // go1.22.6 + "RotateLeft16": true, // go1.22.6 + "RotateLeft32": true, // go1.22.6 + "RotateLeft64": true, // go1.22.6 + "RotateLeft8": true, // go1.22.6 + "Sub": true, // go1.22.6 + "Sub64": true, // go1.22.6 + "TrailingZeros16": true, // go1.22.6 + "TrailingZeros32": true, // go1.22.6 + "TrailingZeros64": true, // go1.22.6 + "TrailingZeros8": true, // go1.22.6 }, "runtime": { - "publicationBarrier": true, + "publicationBarrier": true, // go1.22.6 }, "runtime/internal/atomic": { - "And": true, - "And8": true, - "Cas": true, - "Cas64": true, - "CasRel": true, - "Casint32": true, - "Casint64": true, - "Casp1": true, - "Casuintptr": true, - "Load": true, - "Load64": true, - "Load8": true, - "LoadAcq": true, - "LoadAcq64": true, - "LoadAcquintptr": true, - "Loadint32": true, - "Loadint64": true, - "Loadp": true, - "Loaduint": true, - "Loaduintptr": true, - "Or": true, - "Or8": true, - "Store": true, - "Store64": true, - "Store8": true, - "StoreRel": true, - "StoreRel64": true, - "StoreReluintptr": true, - "Storeint32": true, - "Storeint64": true, - "StorepNoWB": true, - "Storeuintptr": true, - "Xadd": true, - "Xadd64": true, - "Xaddint32": true, - "Xaddint64": true, - "Xadduintptr": true, - "Xchg": true, - "Xchg64": true, - "Xchgint32": true, - "Xchgint64": true, - "Xchguintptr": true, + "And": true, // go1.22.6 + "And8": true, // go1.22.6 + "Cas": true, // go1.22.6 + "Cas64": true, // go1.22.6 + "CasRel": true, // go1.22.6 + "Casint32": true, // go1.22.6 + "Casint64": true, // go1.22.6 + "Casp1": true, // go1.22.6 + "Casuintptr": true, // go1.22.6 + "Load": true, // go1.22.6 + "Load64": true, // go1.22.6 + "Load8": true, // go1.22.6 + "LoadAcq": true, // go1.22.6 + "LoadAcq64": true, // go1.22.6 + "LoadAcquintptr": true, // go1.22.6 + "Loadint32": true, // go1.22.6 + "Loadint64": true, // go1.22.6 + "Loadp": true, // go1.22.6 + "Loaduint": true, // go1.22.6 + "Loaduintptr": true, // go1.22.6 + "Or": true, // go1.22.6 + "Or8": true, // go1.22.6 + "Store": true, // go1.22.6 + "Store64": true, // go1.22.6 + "Store8": true, // go1.22.6 + "StoreRel": true, // go1.22.6 + "StoreRel64": true, // go1.22.6 + "StoreReluintptr": true, // go1.22.6 + "Storeint32": true, // go1.22.6 + "Storeint64": true, // go1.22.6 + "StorepNoWB": true, // go1.22.6 + "Storeuintptr": true, // go1.22.6 + "Xadd": true, // go1.22.6 + "Xadd64": true, // go1.22.6 + "Xaddint32": true, // go1.22.6 + "Xaddint64": true, // go1.22.6 + "Xadduintptr": true, // go1.22.6 + "Xchg": true, // go1.22.6 + "Xchg64": true, // go1.22.6 + "Xchgint32": true, // go1.22.6 + "Xchgint64": true, // go1.22.6 + "Xchguintptr": true, // go1.22.6 }, "runtime/internal/math": { - "MulUintptr": true, + "MulUintptr": true, // go1.22.6 }, "runtime/internal/sys": { - "Bswap32": true, - "Bswap64": true, - "Len64": true, - "Len8": true, - "OnesCount64": true, - "Prefetch": true, - "PrefetchStreamed": true, - "TrailingZeros32": true, - "TrailingZeros64": true, - "TrailingZeros8": true, + "Bswap32": true, // go1.22.6 + "Bswap64": true, // go1.22.6 + "Len64": true, // go1.22.6 + "Len8": true, // go1.22.6 + "OnesCount64": true, // go1.22.6 + "Prefetch": true, // go1.22.6 + "PrefetchStreamed": true, // go1.22.6 + "TrailingZeros32": true, // go1.22.6 + "TrailingZeros64": true, // go1.22.6 + "TrailingZeros8": true, // go1.22.6 }, "sync": { - "runtime_LoadAcquintptr": true, - "runtime_StoreReluintptr": true, + "runtime_LoadAcquintptr": true, // go1.22.6 + "runtime_StoreReluintptr": true, // go1.22.6 }, "sync/atomic": { - "AddInt32": true, - "AddInt64": true, - "AddUint32": true, - "AddUint64": true, - "AddUintptr": true, - "CompareAndSwapInt32": true, - "CompareAndSwapInt64": true, - "CompareAndSwapUint32": true, - "CompareAndSwapUint64": true, - "CompareAndSwapUintptr": true, - "LoadInt32": true, - "LoadInt64": true, - "LoadPointer": true, - "LoadUint32": true, - "LoadUint64": true, - "LoadUintptr": true, - "StoreInt32": true, - "StoreInt64": true, - "StoreUint32": true, - "StoreUint64": true, - "StoreUintptr": true, - "SwapInt32": true, - "SwapInt64": true, - "SwapUint32": true, - "SwapUint64": true, - "SwapUintptr": true, + "AddInt32": true, // go1.22.6 + "AddInt64": true, // go1.22.6 + "AddUint32": true, // go1.22.6 + "AddUint64": true, // go1.22.6 + "AddUintptr": true, // go1.22.6 + "CompareAndSwapInt32": true, // go1.22.6 + "CompareAndSwapInt64": true, // go1.22.6 + "CompareAndSwapUint32": true, // go1.22.6 + "CompareAndSwapUint64": true, // go1.22.6 + "CompareAndSwapUintptr": true, // go1.22.6 + "LoadInt32": true, // go1.22.6 + "LoadInt64": true, // go1.22.6 + "LoadPointer": true, // go1.22.6 + "LoadUint32": true, // go1.22.6 + "LoadUint64": true, // go1.22.6 + "LoadUintptr": true, // go1.22.6 + "StoreInt32": true, // go1.22.6 + "StoreInt64": true, // go1.22.6 + "StoreUint32": true, // go1.22.6 + "StoreUint64": true, // go1.22.6 + "StoreUintptr": true, // go1.22.6 + "SwapInt32": true, // go1.22.6 + "SwapInt64": true, // go1.22.6 + "SwapUint32": true, // go1.22.6 + "SwapUint64": true, // go1.22.6 + "SwapUintptr": true, // go1.22.6 }, } diff --git a/scripts/gen_go_std_tables.go b/scripts/gen_go_std_tables.go index a602bb6..9ab962e 100644 --- a/scripts/gen_go_std_tables.go +++ b/scripts/gen_go_std_tables.go @@ -20,22 +20,24 @@ import ( "text/template" ) +var goVersions = []string{"go1.22.6"} + var tmplTables = template.Must(template.New("").Parse(` // Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT. -// Generated from Go version {{ .GoVersion }}. +// Generated from Go versions {{ .GoVersions }}. package main var runtimeAndDeps = map[string]bool{ {{- range $path := .RuntimeAndDeps }} - "{{ $path }}": true, + "{{ $path.String }}": true, // {{ $path.GoVersion }} {{- end }} } var runtimeLinknamed = []string{ {{- range $path := .RuntimeLinknamed }} - "{{ $path }}", + "{{ $path.String }}", // {{ $path.GoVersion }} {{- end }} // The net package linknames to the runtime, not the other way around. // TODO: support this automatically via our script. @@ -46,7 +48,7 @@ var compilerIntrinsics = map[string]map[string]bool{ {{- range $intr := .CompilerIntrinsics }} "{{ $intr.Path }}": { {{- range $name := $intr.Names }} - "{{ $name }}": true, + "{{ $name.String }}": true, // {{ $name.GoVersion }} {{- end }} }, {{- end }} @@ -58,15 +60,15 @@ var reflectSkipPkg = map[string]bool{ `[1:])) type tmplData struct { - GoVersion string - RuntimeAndDeps []string - RuntimeLinknamed []string + GoVersions []string + RuntimeAndDeps []versionedString + RuntimeLinknamed []versionedString CompilerIntrinsics []tmplIntrinsic } type tmplIntrinsic struct { Path string - Names []string + Names []versionedString } func (t tmplIntrinsic) Compare(t2 tmplIntrinsic) int { @@ -77,13 +79,36 @@ func (t tmplIntrinsic) Equal(t2 tmplIntrinsic) bool { return t.Compare(t2) == 0 } -func cmdGo(args ...string) string { +type versionedString struct { + String string + GoVersion string +} + +func (v versionedString) Compare(v2 versionedString) int { + if c := cmp.Compare(v.String, v2.String); c != 0 { + return c + } + // Negated so that newer Go versions go first. + return -cmp.Compare(v.GoVersion, v2.GoVersion) +} + +func (v versionedString) Equal(v2 versionedString) bool { + // Note that we do equality based on String alone, + // because we only need one String entry with the latest version. + return v.String == v2.String +} + +func cmdGo(goVersion string, args ...string) versionedString { cmd := exec.Command("go", args...) + cmd.Env = append(cmd.Environ(), "GOTOOLCHAIN="+goVersion) out, err := cmd.Output() if err != nil { panic(err) } - return string(bytes.TrimSpace(out)) // no trailing newline + return versionedString{ + String: string(bytes.TrimSpace(out)), // no trailing newline + GoVersion: goVersion, + } } func readFile(path string) string { @@ -94,81 +119,109 @@ func readFile(path string) string { return string(data) } -func sortedLines(s string) []string { - lines := strings.Split(s, "\n") - slices.Sort(lines) - lines = slices.Compact(lines) - return lines +func lines(vs versionedString) []versionedString { + split := strings.Split(vs.String, "\n") + var versioned []versionedString + for _, s := range split { + versioned = append(versioned, versionedString{ + String: s, + GoVersion: vs.GoVersion, + }) + } + return versioned } var rxLinkname = regexp.MustCompile(`^//go:linkname .* ([^.]*)\.[^.]*$`) var rxIntrinsic = regexp.MustCompile(`\b(addF|alias)\("([^"]*)", "([^"]*)",`) func main() { - goversion := cmdGo("env", "GOVERSION") // not "go version", to exclude GOOS/GOARCH - goroot := cmdGo("env", "GOROOT") + var runtimeAndDeps []versionedString + for _, goVersion := range goVersions { + runtimeAndDeps = append(runtimeAndDeps, lines(cmdGo(goVersion, "list", "-deps", "runtime"))...) + } + slices.SortFunc(runtimeAndDeps, versionedString.Compare) + runtimeAndDeps = slices.CompactFunc(runtimeAndDeps, versionedString.Equal) - runtimeAndDeps := sortedLines(cmdGo("list", "-deps", "runtime")) + var goroots []versionedString + for _, goVersion := range goVersions { + goroots = append(goroots, cmdGo(goVersion, "env", "GOROOT")) + } // All packages that the runtime linknames to, except runtime and its dependencies. // This resulting list is what we need to "go list" when obfuscating the runtime, // as they are the packages that we may be missing. - var runtimeLinknamed []string - runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot, "src", "runtime", "*.go")) - if err != nil { - panic(err) - } - for _, goFile := range runtimeGoFiles { - for _, line := range strings.Split(readFile(goFile), "\n") { - m := rxLinkname.FindStringSubmatch(line) - if m == nil { - continue - } - path := m[1] - switch path { - case "main", "runtime/metrics_test": - continue + var runtimeLinknamed []versionedString + for _, goroot := range goroots { + runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot.String, "src", "runtime", "*.go")) + if err != nil { + panic(err) + } + for _, goFile := range runtimeGoFiles { + for _, line := range strings.Split(readFile(goFile), "\n") { + m := rxLinkname.FindStringSubmatch(line) + if m == nil { + continue + } + path := m[1] + switch path { + case "main", "runtime/metrics_test": + continue + } + runtimeLinknamed = append(runtimeLinknamed, versionedString{ + String: path, + GoVersion: goroot.GoVersion, + }) } - runtimeLinknamed = append(runtimeLinknamed, path) } } - slices.Sort(runtimeLinknamed) - runtimeLinknamed = slices.Compact(runtimeLinknamed) - runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path string) bool { - return slices.Contains(runtimeAndDeps, path) + slices.SortFunc(runtimeLinknamed, versionedString.Compare) + runtimeLinknamed = slices.CompactFunc(runtimeLinknamed, versionedString.Equal) + runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path versionedString) bool { + for _, prev := range runtimeAndDeps { + if prev.String == path.String { + return true + } + } + return false }) compilerIntrinsicsIndexByPath := make(map[string]int) var compilerIntrinsics []tmplIntrinsic - for _, line := range strings.Split(readFile(filepath.Join( - goroot, "src", "cmd", "compile", "internal", "ssagen", "ssa.go", - )), "\n") { - m := rxIntrinsic.FindStringSubmatch(line) - if m == nil { - continue - } - path, name := m[2], m[3] - if i := compilerIntrinsicsIndexByPath[path]; i == 0 { - compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics) - compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{ - Path: path, - Names: []string{name}, - }) - } else { - compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, name) + for _, goroot := range goroots { + for _, line := range strings.Split(readFile(filepath.Join( + goroot.String, "src", "cmd", "compile", "internal", "ssagen", "ssa.go", + )), "\n") { + m := rxIntrinsic.FindStringSubmatch(line) + if m == nil { + continue + } + path, name := m[2], m[3] + vs := versionedString{ + String: name, + GoVersion: goroot.GoVersion, + } + if i := compilerIntrinsicsIndexByPath[path]; i == 0 { + compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics) + compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{ + Path: path, + Names: []versionedString{vs}, + }) + } else { + compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, vs) + } } } slices.SortFunc(compilerIntrinsics, tmplIntrinsic.Compare) compilerIntrinsics = slices.CompactFunc(compilerIntrinsics, tmplIntrinsic.Equal) for path := range compilerIntrinsics { intr := &compilerIntrinsics[path] - slices.Sort(intr.Names) - intr.Names = slices.Compact(intr.Names) + slices.SortFunc(intr.Names, versionedString.Compare) + intr.Names = slices.CompactFunc(intr.Names, versionedString.Equal) } var buf bytes.Buffer if err := tmplTables.Execute(&buf, tmplData{ - GoVersion: goversion, + GoVersions: goVersions, RuntimeAndDeps: runtimeAndDeps, RuntimeLinknamed: runtimeLinknamed, CompilerIntrinsics: compilerIntrinsics,