justan0th3rstud3nt
/
garble
mirror of https://github.com/burrowers/garble
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

generate Go tables with a list of GOTOOLCHAIN versions

Browse Source 
This teaches the program how to collect information from multiple
Go versions and join it together. For this to work, it needs to
select the Go versions itself, which is now possible via GOTOOLCHAIN.

The merging of data is fairly simple; we join the results from all
versions, and we remove duplicates from older Go versions.

Start producing output with the Go version noted on every data point,
so that we can easily scan what each Go version is contributing.
pull/876/head
Daniel Martí 7 months ago
parent b49a13c556
commit c1439947f9
2 changed files with 278 additions and 225 deletions
Show Stats Download Patch File Download Diff File

336
go_std_tables.go
Unescape Escape View File

@ -1,53 +1,53 @@
// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT. // Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
// Generated from Go version go1.22.6. // Generated from Go versions [go1.22.6].
package main package main
var runtimeAndDeps = map[string]bool{ var runtimeAndDeps = map[string]bool{
"internal/abi": true, "internal/abi": true, // go1.22.6
"internal/bytealg": true, "internal/bytealg": true, // go1.22.6
"internal/chacha8rand": true, "internal/chacha8rand": true, // go1.22.6
"internal/coverage/rtcov": true, "internal/coverage/rtcov": true, // go1.22.6
"internal/cpu": true, "internal/cpu": true, // go1.22.6
"internal/goarch": true, "internal/goarch": true, // go1.22.6
"internal/godebugs": true, "internal/godebugs": true, // go1.22.6
"internal/goexperiment": true, "internal/goexperiment": true, // go1.22.6
"internal/goos": true, "internal/goos": true, // go1.22.6
"runtime": true, "runtime": true, // go1.22.6
"runtime/internal/atomic": true, "runtime/internal/atomic": true, // go1.22.6
"runtime/internal/math": true, "runtime/internal/math": true, // go1.22.6
"runtime/internal/sys": true, "runtime/internal/sys": true, // go1.22.6
"runtime/internal/syscall": true, "runtime/internal/syscall": true, // go1.22.6
"unsafe": true, "unsafe": true, // go1.22.6
} }
var runtimeLinknamed = []string{ var runtimeLinknamed = []string{
"arena", "arena", // go1.22.6
"crypto/internal/boring", "crypto/internal/boring", // go1.22.6
"crypto/internal/boring/bcache", "crypto/internal/boring/bcache", // go1.22.6
"crypto/internal/boring/fipstls", "crypto/internal/boring/fipstls", // go1.22.6
"crypto/x509/internal/macos", "crypto/x509/internal/macos", // go1.22.6
"internal/godebug", "internal/godebug", // go1.22.6
"internal/poll", "internal/poll", // go1.22.6
"internal/reflectlite", "internal/reflectlite", // go1.22.6
"internal/syscall/unix", "internal/syscall/unix", // go1.22.6
"internal/syscall/windows", "internal/syscall/windows", // go1.22.6
"maps", "maps", // go1.22.6
"os", "os", // go1.22.6
"os/signal", "os/signal", // go1.22.6
"plugin", "plugin", // go1.22.6
"reflect", "reflect", // go1.22.6
"runtime/coverage", "runtime/coverage", // go1.22.6
"runtime/debug", "runtime/debug", // go1.22.6
"runtime/metrics", "runtime/metrics", // go1.22.6
"runtime/pprof", "runtime/pprof", // go1.22.6
"runtime/trace", "runtime/trace", // go1.22.6
"sync", "sync", // go1.22.6
"sync/atomic", "sync/atomic", // go1.22.6
"syscall", "syscall", // go1.22.6
"syscall/js", "syscall/js", // go1.22.6
"time", "time", // go1.22.6
// The net package linknames to the runtime, not the other way around. // The net package linknames to the runtime, not the other way around.
// TODO: support this automatically via our script. // TODO: support this automatically via our script.
"net", "net",
@ -55,149 +55,149 @@ var runtimeLinknamed = []string{
var compilerIntrinsics = map[string]map[string]bool{ var compilerIntrinsics = map[string]map[string]bool{
"math": { "math": {
"Abs": true, "Abs": true, // go1.22.6
"Ceil": true, "Ceil": true, // go1.22.6
"Copysign": true, "Copysign": true, // go1.22.6
"FMA": true, "FMA": true, // go1.22.6
"Floor": true, "Floor": true, // go1.22.6
"Round": true, "Round": true, // go1.22.6
"RoundToEven": true, "RoundToEven": true, // go1.22.6
"Trunc": true, "Trunc": true, // go1.22.6
"sqrt": true, "sqrt": true, // go1.22.6
}, },
"math/big": { "math/big": {
"mulWW": true, "mulWW": true, // go1.22.6
}, },
"math/bits": { "math/bits": {
"Add": true, "Add": true, // go1.22.6
"Add64": true, "Add64": true, // go1.22.6
"Div": true, "Div": true, // go1.22.6
"Div64": true, "Div64": true, // go1.22.6
"Len": true, "Len": true, // go1.22.6
"Len16": true, "Len16": true, // go1.22.6
"Len32": true, "Len32": true, // go1.22.6
"Len64": true, "Len64": true, // go1.22.6
"Len8": true, "Len8": true, // go1.22.6
"Mul": true, "Mul": true, // go1.22.6
"Mul64": true, "Mul64": true, // go1.22.6
"OnesCount": true, "OnesCount": true, // go1.22.6
"OnesCount16": true, "OnesCount16": true, // go1.22.6
"OnesCount32": true, "OnesCount32": true, // go1.22.6
"OnesCount64": true, "OnesCount64": true, // go1.22.6
"OnesCount8": true, "OnesCount8": true, // go1.22.6
"Reverse": true, "Reverse": true, // go1.22.6
"Reverse16": true, "Reverse16": true, // go1.22.6
"Reverse32": true, "Reverse32": true, // go1.22.6
"Reverse64": true, "Reverse64": true, // go1.22.6
"Reverse8": true, "Reverse8": true, // go1.22.6
"ReverseBytes16": true, "ReverseBytes16": true, // go1.22.6
"ReverseBytes32": true, "ReverseBytes32": true, // go1.22.6
"ReverseBytes64": true, "ReverseBytes64": true, // go1.22.6
"RotateLeft": true, "RotateLeft": true, // go1.22.6
"RotateLeft16": true, "RotateLeft16": true, // go1.22.6
"RotateLeft32": true, "RotateLeft32": true, // go1.22.6
"RotateLeft64": true, "RotateLeft64": true, // go1.22.6
"RotateLeft8": true, "RotateLeft8": true, // go1.22.6
"Sub": true, "Sub": true, // go1.22.6
"Sub64": true, "Sub64": true, // go1.22.6
"TrailingZeros16": true, "TrailingZeros16": true, // go1.22.6
"TrailingZeros32": true, "TrailingZeros32": true, // go1.22.6
"TrailingZeros64": true, "TrailingZeros64": true, // go1.22.6
"TrailingZeros8": true, "TrailingZeros8": true, // go1.22.6
}, },
"runtime": { "runtime": {
"publicationBarrier": true, "publicationBarrier": true, // go1.22.6
}, },
"runtime/internal/atomic": { "runtime/internal/atomic": {
"And": true, "And": true, // go1.22.6
"And8": true, "And8": true, // go1.22.6
"Cas": true, "Cas": true, // go1.22.6
"Cas64": true, "Cas64": true, // go1.22.6
"CasRel": true, "CasRel": true, // go1.22.6
"Casint32": true, "Casint32": true, // go1.22.6
"Casint64": true, "Casint64": true, // go1.22.6
"Casp1": true, "Casp1": true, // go1.22.6
"Casuintptr": true, "Casuintptr": true, // go1.22.6
"Load": true, "Load": true, // go1.22.6
"Load64": true, "Load64": true, // go1.22.6
"Load8": true, "Load8": true, // go1.22.6
"LoadAcq": true, "LoadAcq": true, // go1.22.6
"LoadAcq64": true, "LoadAcq64": true, // go1.22.6
"LoadAcquintptr": true, "LoadAcquintptr": true, // go1.22.6
"Loadint32": true, "Loadint32": true, // go1.22.6
"Loadint64": true, "Loadint64": true, // go1.22.6
"Loadp": true, "Loadp": true, // go1.22.6
"Loaduint": true, "Loaduint": true, // go1.22.6
"Loaduintptr": true, "Loaduintptr": true, // go1.22.6
"Or": true, "Or": true, // go1.22.6
"Or8": true, "Or8": true, // go1.22.6
"Store": true, "Store": true, // go1.22.6
"Store64": true, "Store64": true, // go1.22.6
"Store8": true, "Store8": true, // go1.22.6
"StoreRel": true, "StoreRel": true, // go1.22.6
"StoreRel64": true, "StoreRel64": true, // go1.22.6
"StoreReluintptr": true, "StoreReluintptr": true, // go1.22.6
"Storeint32": true, "Storeint32": true, // go1.22.6
"Storeint64": true, "Storeint64": true, // go1.22.6
"StorepNoWB": true, "StorepNoWB": true, // go1.22.6
"Storeuintptr": true, "Storeuintptr": true, // go1.22.6
"Xadd": true, "Xadd": true, // go1.22.6
"Xadd64": true, "Xadd64": true, // go1.22.6
"Xaddint32": true, "Xaddint32": true, // go1.22.6
"Xaddint64": true, "Xaddint64": true, // go1.22.6
"Xadduintptr": true, "Xadduintptr": true, // go1.22.6
"Xchg": true, "Xchg": true, // go1.22.6
"Xchg64": true, "Xchg64": true, // go1.22.6
"Xchgint32": true, "Xchgint32": true, // go1.22.6
"Xchgint64": true, "Xchgint64": true, // go1.22.6
"Xchguintptr": true, "Xchguintptr": true, // go1.22.6
}, },
"runtime/internal/math": { "runtime/internal/math": {
"MulUintptr": true, "MulUintptr": true, // go1.22.6
}, },
"runtime/internal/sys": { "runtime/internal/sys": {
"Bswap32": true, "Bswap32": true, // go1.22.6
"Bswap64": true, "Bswap64": true, // go1.22.6
"Len64": true, "Len64": true, // go1.22.6
"Len8": true, "Len8": true, // go1.22.6
"OnesCount64": true, "OnesCount64": true, // go1.22.6
"Prefetch": true, "Prefetch": true, // go1.22.6
"PrefetchStreamed": true, "PrefetchStreamed": true, // go1.22.6
"TrailingZeros32": true, "TrailingZeros32": true, // go1.22.6
"TrailingZeros64": true, "TrailingZeros64": true, // go1.22.6
"TrailingZeros8": true, "TrailingZeros8": true, // go1.22.6
}, },
"sync": { "sync": {
"runtime_LoadAcquintptr": true, "runtime_LoadAcquintptr": true, // go1.22.6
"runtime_StoreReluintptr": true, "runtime_StoreReluintptr": true, // go1.22.6
}, },
"sync/atomic": { "sync/atomic": {
"AddInt32": true, "AddInt32": true, // go1.22.6
"AddInt64": true, "AddInt64": true, // go1.22.6
"AddUint32": true, "AddUint32": true, // go1.22.6
"AddUint64": true, "AddUint64": true, // go1.22.6
"AddUintptr": true, "AddUintptr": true, // go1.22.6
"CompareAndSwapInt32": true, "CompareAndSwapInt32": true, // go1.22.6
"CompareAndSwapInt64": true, "CompareAndSwapInt64": true, // go1.22.6
"CompareAndSwapUint32": true, "CompareAndSwapUint32": true, // go1.22.6
"CompareAndSwapUint64": true, "CompareAndSwapUint64": true, // go1.22.6
"CompareAndSwapUintptr": true, "CompareAndSwapUintptr": true, // go1.22.6
"LoadInt32": true, "LoadInt32": true, // go1.22.6
"LoadInt64": true, "LoadInt64": true, // go1.22.6
"LoadPointer": true, "LoadPointer": true, // go1.22.6
"LoadUint32": true, "LoadUint32": true, // go1.22.6
"LoadUint64": true, "LoadUint64": true, // go1.22.6
"LoadUintptr": true, "LoadUintptr": true, // go1.22.6
"StoreInt32": true, "StoreInt32": true, // go1.22.6
"StoreInt64": true, "StoreInt64": true, // go1.22.6
"StoreUint32": true, "StoreUint32": true, // go1.22.6
"StoreUint64": true, "StoreUint64": true, // go1.22.6
"StoreUintptr": true, "StoreUintptr": true, // go1.22.6
"SwapInt32": true, "SwapInt32": true, // go1.22.6
"SwapInt64": true, "SwapInt64": true, // go1.22.6
"SwapUint32": true, "SwapUint32": true, // go1.22.6
"SwapUint64": true, "SwapUint64": true, // go1.22.6
"SwapUintptr": true, "SwapUintptr": true, // go1.22.6
}, },
} }

167
scripts/gen_go_std_tables.go
Unescape Escape View File

@ -20,22 +20,24 @@ import (
"text/template" "text/template"
) )
var goVersions = []string{"go1.22.6"}
var tmplTables = template.Must(template.New("").Parse(` var tmplTables = template.Must(template.New("").Parse(`
// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT. // Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
// Generated from Go version {{ .GoVersion }}. // Generated from Go versions {{ .GoVersions }}.
package main package main
var runtimeAndDeps = map[string]bool{ var runtimeAndDeps = map[string]bool{
{{- range $path := .RuntimeAndDeps }} {{- range $path := .RuntimeAndDeps }}
"{{ $path }}": true, "{{ $path.String }}": true, // {{ $path.GoVersion }}
{{- end }} {{- end }}
} }
var runtimeLinknamed = []string{ var runtimeLinknamed = []string{
{{- range $path := .RuntimeLinknamed }} {{- range $path := .RuntimeLinknamed }}
"{{ $path }}", "{{ $path.String }}", // {{ $path.GoVersion }}
{{- end }} {{- end }}
// The net package linknames to the runtime, not the other way around. // The net package linknames to the runtime, not the other way around.
// TODO: support this automatically via our script. // TODO: support this automatically via our script.
@ -46,7 +48,7 @@ var compilerIntrinsics = map[string]map[string]bool{
{{- range $intr := .CompilerIntrinsics }} {{- range $intr := .CompilerIntrinsics }}
"{{ $intr.Path }}": { "{{ $intr.Path }}": {
{{- range $name := $intr.Names }} {{- range $name := $intr.Names }}
"{{ $name }}": true, "{{ $name.String }}": true, // {{ $name.GoVersion }}
{{- end }} {{- end }}
}, },
{{- end }} {{- end }}
@ -58,15 +60,15 @@ var reflectSkipPkg = map[string]bool{
`[1:])) `[1:]))
type tmplData struct { type tmplData struct {
GoVersion string GoVersions []string
RuntimeAndDeps []string RuntimeAndDeps []versionedString
RuntimeLinknamed []string RuntimeLinknamed []versionedString
CompilerIntrinsics []tmplIntrinsic CompilerIntrinsics []tmplIntrinsic
} }
type tmplIntrinsic struct { type tmplIntrinsic struct {
Path string Path string
Names []string Names []versionedString
} }
func (t tmplIntrinsic) Compare(t2 tmplIntrinsic) int { func (t tmplIntrinsic) Compare(t2 tmplIntrinsic) int {
@ -77,13 +79,36 @@ func (t tmplIntrinsic) Equal(t2 tmplIntrinsic) bool {
return t.Compare(t2) == 0 return t.Compare(t2) == 0
} }
func cmdGo(args ...string) string { type versionedString struct {
String string
GoVersion string
}
func (v versionedString) Compare(v2 versionedString) int {
if c := cmp.Compare(v.String, v2.String); c != 0 {
return c
}
// Negated so that newer Go versions go first.
return -cmp.Compare(v.GoVersion, v2.GoVersion)
}
func (v versionedString) Equal(v2 versionedString) bool {
// Note that we do equality based on String alone,
// because we only need one String entry with the latest version.
return v.String == v2.String
}
func cmdGo(goVersion string, args ...string) versionedString {
cmd := exec.Command("go", args...) cmd := exec.Command("go", args...)
cmd.Env = append(cmd.Environ(), "GOTOOLCHAIN="+goVersion)
out, err := cmd.Output() out, err := cmd.Output()
if err != nil { if err != nil {
panic(err) panic(err)
} }
return string(bytes.TrimSpace(out)) // no trailing newline return versionedString{
String: string(bytes.TrimSpace(out)), // no trailing newline
GoVersion: goVersion,
}
} }
func readFile(path string) string { func readFile(path string) string {
@ -94,81 +119,109 @@ func readFile(path string) string {
return string(data) return string(data)
} }
func sortedLines(s string) []string { func lines(vs versionedString) []versionedString {
lines := strings.Split(s, "\n") split := strings.Split(vs.String, "\n")
slices.Sort(lines) var versioned []versionedString
lines = slices.Compact(lines) for _, s := range split {
return lines versioned = append(versioned, versionedString{
String: s,
GoVersion: vs.GoVersion,
})
}
return versioned
} }
var rxLinkname = regexp.MustCompile(`^//go:linkname .* ([^.]*)\.[^.]*$`) var rxLinkname = regexp.MustCompile(`^//go:linkname .* ([^.]*)\.[^.]*$`)
var rxIntrinsic = regexp.MustCompile(`\b(addF|alias)\("([^"]*)", "([^"]*)",`) var rxIntrinsic = regexp.MustCompile(`\b(addF|alias)\("([^"]*)", "([^"]*)",`)
func main() { func main() {
goversion := cmdGo("env", "GOVERSION") // not "go version", to exclude GOOS/GOARCH var runtimeAndDeps []versionedString
goroot := cmdGo("env", "GOROOT") for _, goVersion := range goVersions {
runtimeAndDeps = append(runtimeAndDeps, lines(cmdGo(goVersion, "list", "-deps", "runtime"))...)
}
slices.SortFunc(runtimeAndDeps, versionedString.Compare)
runtimeAndDeps = slices.CompactFunc(runtimeAndDeps, versionedString.Equal)
runtimeAndDeps := sortedLines(cmdGo("list", "-deps", "runtime")) var goroots []versionedString
for _, goVersion := range goVersions {
goroots = append(goroots, cmdGo(goVersion, "env", "GOROOT"))
}
// All packages that the runtime linknames to, except runtime and its dependencies. // All packages that the runtime linknames to, except runtime and its dependencies.
// This resulting list is what we need to "go list" when obfuscating the runtime, // This resulting list is what we need to "go list" when obfuscating the runtime,
// as they are the packages that we may be missing. // as they are the packages that we may be missing.
var runtimeLinknamed []string var runtimeLinknamed []versionedString
runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot, "src", "runtime", "*.go")) for _, goroot := range goroots {
if err != nil { runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot.String, "src", "runtime", "*.go"))
panic(err) if err != nil {
} panic(err)
for _, goFile := range runtimeGoFiles { }
for _, line := range strings.Split(readFile(goFile), "\n") { for _, goFile := range runtimeGoFiles {
m := rxLinkname.FindStringSubmatch(line) for _, line := range strings.Split(readFile(goFile), "\n") {
if m == nil { m := rxLinkname.FindStringSubmatch(line)
continue if m == nil {
} continue
path := m[1] }
switch path { path := m[1]
case "main", "runtime/metrics_test": switch path {
continue case "main", "runtime/metrics_test":
continue
}
runtimeLinknamed = append(runtimeLinknamed, versionedString{
String: path,
GoVersion: goroot.GoVersion,
})
} }
runtimeLinknamed = append(runtimeLinknamed, path)
} }
} }
slices.Sort(runtimeLinknamed) slices.SortFunc(runtimeLinknamed, versionedString.Compare)
runtimeLinknamed = slices.Compact(runtimeLinknamed) runtimeLinknamed = slices.CompactFunc(runtimeLinknamed, versionedString.Equal)
runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path string) bool { runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path versionedString) bool {
return slices.Contains(runtimeAndDeps, path) for _, prev := range runtimeAndDeps {
if prev.String == path.String {
return true
}
}
return false
}) })
compilerIntrinsicsIndexByPath := make(map[string]int) compilerIntrinsicsIndexByPath := make(map[string]int)
var compilerIntrinsics []tmplIntrinsic var compilerIntrinsics []tmplIntrinsic
for _, line := range strings.Split(readFile(filepath.Join( for _, goroot := range goroots {
goroot, "src", "cmd", "compile", "internal", "ssagen", "ssa.go", for _, line := range strings.Split(readFile(filepath.Join(
)), "\n") { goroot.String, "src", "cmd", "compile", "internal", "ssagen", "ssa.go",
m := rxIntrinsic.FindStringSubmatch(line) )), "\n") {
if m == nil { m := rxIntrinsic.FindStringSubmatch(line)
continue if m == nil {
} continue
path, name := m[2], m[3] }
if i := compilerIntrinsicsIndexByPath[path]; i == 0 { path, name := m[2], m[3]
compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics) vs := versionedString{
compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{ String: name,
Path: path, GoVersion: goroot.GoVersion,
Names: []string{name}, }
}) if i := compilerIntrinsicsIndexByPath[path]; i == 0 {
} else { compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics)
compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, name) compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{
Path: path,
Names: []versionedString{vs},
})
} else {
compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, vs)
}
} }
} }
slices.SortFunc(compilerIntrinsics, tmplIntrinsic.Compare) slices.SortFunc(compilerIntrinsics, tmplIntrinsic.Compare)
compilerIntrinsics = slices.CompactFunc(compilerIntrinsics, tmplIntrinsic.Equal) compilerIntrinsics = slices.CompactFunc(compilerIntrinsics, tmplIntrinsic.Equal)
for path := range compilerIntrinsics { for path := range compilerIntrinsics {
intr := &compilerIntrinsics[path] intr := &compilerIntrinsics[path]
slices.Sort(intr.Names) slices.SortFunc(intr.Names, versionedString.Compare)
intr.Names = slices.Compact(intr.Names) intr.Names = slices.CompactFunc(intr.Names, versionedString.Equal)
} }
var buf bytes.Buffer var buf bytes.Buffer
if err := tmplTables.Execute(&buf, tmplData{ if err := tmplTables.Execute(&buf, tmplData{
GoVersion: goversion, GoVersions: goVersions,
RuntimeAndDeps: runtimeAndDeps, RuntimeAndDeps: runtimeAndDeps,
RuntimeLinknamed: runtimeLinknamed, RuntimeLinknamed: runtimeLinknamed,
CompilerIntrinsics: compilerIntrinsics, CompilerIntrinsics: compilerIntrinsics,

Write Preview
Loading…
Cancel
Save