justan0th3rstud3nt
/
garble
mirror of https://github.com/burrowers/garble
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

generate Go tables with a list of GOTOOLCHAIN versions

Browse Source 
This teaches the program how to collect information from multiple
Go versions and join it together. For this to work, it needs to
select the Go versions itself, which is now possible via GOTOOLCHAIN.

The merging of data is fairly simple; we join the results from all
versions, and we remove duplicates from older Go versions.

Start producing output with the Go version noted on every data point,
so that we can easily scan what each Go version is contributing.
pull/876/head
Daniel Martí 7 months ago
parent b49a13c556
commit c1439947f9
2 changed files with 278 additions and 225 deletions
Show Stats Download Patch File Download Diff File

336
go_std_tables.go
Unescape Escape View File

@ -1,53 +1,53 @@
// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
// Generated from Go version go1.22.6.
// Generated from Go versions [go1.22.6].
package main
var runtimeAndDeps = map[string]bool{
"internal/abi": true,
"internal/bytealg": true,
"internal/chacha8rand": true,
"internal/coverage/rtcov": true,
"internal/cpu": true,
"internal/goarch": true,
"internal/godebugs": true,
"internal/goexperiment": true,
"internal/goos": true,
"runtime": true,
"runtime/internal/atomic": true,
"runtime/internal/math": true,
"runtime/internal/sys": true,
"runtime/internal/syscall": true,
"unsafe": true,
"internal/abi": true, // go1.22.6
"internal/bytealg": true, // go1.22.6
"internal/chacha8rand": true, // go1.22.6
"internal/coverage/rtcov": true, // go1.22.6
"internal/cpu": true, // go1.22.6
"internal/goarch": true, // go1.22.6
"internal/godebugs": true, // go1.22.6
"internal/goexperiment": true, // go1.22.6
"internal/goos": true, // go1.22.6
"runtime": true, // go1.22.6
"runtime/internal/atomic": true, // go1.22.6
"runtime/internal/math": true, // go1.22.6
"runtime/internal/sys": true, // go1.22.6
"runtime/internal/syscall": true, // go1.22.6
"unsafe": true, // go1.22.6
}
var runtimeLinknamed = []string{
"arena",
"crypto/internal/boring",
"crypto/internal/boring/bcache",
"crypto/internal/boring/fipstls",
"crypto/x509/internal/macos",
"internal/godebug",
"internal/poll",
"internal/reflectlite",
"internal/syscall/unix",
"internal/syscall/windows",
"maps",
"os",
"os/signal",
"plugin",
"reflect",
"runtime/coverage",
"runtime/debug",
"runtime/metrics",
"runtime/pprof",
"runtime/trace",
"sync",
"sync/atomic",
"syscall",
"syscall/js",
"time",
"arena", // go1.22.6
"crypto/internal/boring", // go1.22.6
"crypto/internal/boring/bcache", // go1.22.6
"crypto/internal/boring/fipstls", // go1.22.6
"crypto/x509/internal/macos", // go1.22.6
"internal/godebug", // go1.22.6
"internal/poll", // go1.22.6
"internal/reflectlite", // go1.22.6
"internal/syscall/unix", // go1.22.6
"internal/syscall/windows", // go1.22.6
"maps", // go1.22.6
"os", // go1.22.6
"os/signal", // go1.22.6
"plugin", // go1.22.6
"reflect", // go1.22.6
"runtime/coverage", // go1.22.6
"runtime/debug", // go1.22.6
"runtime/metrics", // go1.22.6
"runtime/pprof", // go1.22.6
"runtime/trace", // go1.22.6
"sync", // go1.22.6
"sync/atomic", // go1.22.6
"syscall", // go1.22.6
"syscall/js", // go1.22.6
"time", // go1.22.6
// The net package linknames to the runtime, not the other way around.
// TODO: support this automatically via our script.
"net",
@ -55,149 +55,149 @@ var runtimeLinknamed = []string{
var compilerIntrinsics = map[string]map[string]bool{
"math": {
"Abs": true,
"Ceil": true,
"Copysign": true,
"FMA": true,
"Floor": true,
"Round": true,
"RoundToEven": true,
"Trunc": true,
"sqrt": true,
"Abs": true, // go1.22.6
"Ceil": true, // go1.22.6
"Copysign": true, // go1.22.6
"FMA": true, // go1.22.6
"Floor": true, // go1.22.6
"Round": true, // go1.22.6
"RoundToEven": true, // go1.22.6
"Trunc": true, // go1.22.6
"sqrt": true, // go1.22.6
},
"math/big": {
"mulWW": true,
"mulWW": true, // go1.22.6
},
"math/bits": {
"Add": true,
"Add64": true,
"Div": true,
"Div64": true,
"Len": true,
"Len16": true,
"Len32": true,
"Len64": true,
"Len8": true,
"Mul": true,
"Mul64": true,
"OnesCount": true,
"OnesCount16": true,
"OnesCount32": true,
"OnesCount64": true,
"OnesCount8": true,
"Reverse": true,
"Reverse16": true,
"Reverse32": true,
"Reverse64": true,
"Reverse8": true,
"ReverseBytes16": true,
"ReverseBytes32": true,
"ReverseBytes64": true,
"RotateLeft": true,
"RotateLeft16": true,
"RotateLeft32": true,
"RotateLeft64": true,
"RotateLeft8": true,
"Sub": true,
"Sub64": true,
"TrailingZeros16": true,
"TrailingZeros32": true,
"TrailingZeros64": true,
"TrailingZeros8": true,
"Add": true, // go1.22.6
"Add64": true, // go1.22.6
"Div": true, // go1.22.6
"Div64": true, // go1.22.6
"Len": true, // go1.22.6
"Len16": true, // go1.22.6
"Len32": true, // go1.22.6
"Len64": true, // go1.22.6
"Len8": true, // go1.22.6
"Mul": true, // go1.22.6
"Mul64": true, // go1.22.6
"OnesCount": true, // go1.22.6
"OnesCount16": true, // go1.22.6
"OnesCount32": true, // go1.22.6
"OnesCount64": true, // go1.22.6
"OnesCount8": true, // go1.22.6
"Reverse": true, // go1.22.6
"Reverse16": true, // go1.22.6
"Reverse32": true, // go1.22.6
"Reverse64": true, // go1.22.6
"Reverse8": true, // go1.22.6
"ReverseBytes16": true, // go1.22.6
"ReverseBytes32": true, // go1.22.6
"ReverseBytes64": true, // go1.22.6
"RotateLeft": true, // go1.22.6
"RotateLeft16": true, // go1.22.6
"RotateLeft32": true, // go1.22.6
"RotateLeft64": true, // go1.22.6
"RotateLeft8": true, // go1.22.6
"Sub": true, // go1.22.6
"Sub64": true, // go1.22.6
"TrailingZeros16": true, // go1.22.6
"TrailingZeros32": true, // go1.22.6
"TrailingZeros64": true, // go1.22.6
"TrailingZeros8": true, // go1.22.6
},
"runtime": {
"publicationBarrier": true,
"publicationBarrier": true, // go1.22.6
},
"runtime/internal/atomic": {
"And": true,
"And8": true,
"Cas": true,
"Cas64": true,
"CasRel": true,
"Casint32": true,
"Casint64": true,
"Casp1": true,
"Casuintptr": true,
"Load": true,
"Load64": true,
"Load8": true,
"LoadAcq": true,
"LoadAcq64": true,
"LoadAcquintptr": true,
"Loadint32": true,
"Loadint64": true,
"Loadp": true,
"Loaduint": true,
"Loaduintptr": true,
"Or": true,
"Or8": true,
"Store": true,
"Store64": true,
"Store8": true,
"StoreRel": true,
"StoreRel64": true,
"StoreReluintptr": true,
"Storeint32": true,
"Storeint64": true,
"StorepNoWB": true,
"Storeuintptr": true,
"Xadd": true,
"Xadd64": true,
"Xaddint32": true,
"Xaddint64": true,
"Xadduintptr": true,
"Xchg": true,
"Xchg64": true,
"Xchgint32": true,
"Xchgint64": true,
"Xchguintptr": true,
"And": true, // go1.22.6
"And8": true, // go1.22.6
"Cas": true, // go1.22.6
"Cas64": true, // go1.22.6
"CasRel": true, // go1.22.6
"Casint32": true, // go1.22.6
"Casint64": true, // go1.22.6
"Casp1": true, // go1.22.6
"Casuintptr": true, // go1.22.6
"Load": true, // go1.22.6
"Load64": true, // go1.22.6
"Load8": true, // go1.22.6
"LoadAcq": true, // go1.22.6
"LoadAcq64": true, // go1.22.6
"LoadAcquintptr": true, // go1.22.6
"Loadint32": true, // go1.22.6
"Loadint64": true, // go1.22.6
"Loadp": true, // go1.22.6
"Loaduint": true, // go1.22.6
"Loaduintptr": true, // go1.22.6
"Or": true, // go1.22.6
"Or8": true, // go1.22.6
"Store": true, // go1.22.6
"Store64": true, // go1.22.6
"Store8": true, // go1.22.6
"StoreRel": true, // go1.22.6
"StoreRel64": true, // go1.22.6
"StoreReluintptr": true, // go1.22.6
"Storeint32": true, // go1.22.6
"Storeint64": true, // go1.22.6
"StorepNoWB": true, // go1.22.6
"Storeuintptr": true, // go1.22.6
"Xadd": true, // go1.22.6
"Xadd64": true, // go1.22.6
"Xaddint32": true, // go1.22.6
"Xaddint64": true, // go1.22.6
"Xadduintptr": true, // go1.22.6
"Xchg": true, // go1.22.6
"Xchg64": true, // go1.22.6
"Xchgint32": true, // go1.22.6
"Xchgint64": true, // go1.22.6
"Xchguintptr": true, // go1.22.6
},
"runtime/internal/math": {
"MulUintptr": true,
"MulUintptr": true, // go1.22.6
},
"runtime/internal/sys": {
"Bswap32": true,
"Bswap64": true,
"Len64": true,
"Len8": true,
"OnesCount64": true,
"Prefetch": true,
"PrefetchStreamed": true,
"TrailingZeros32": true,
"TrailingZeros64": true,
"TrailingZeros8": true,
"Bswap32": true, // go1.22.6
"Bswap64": true, // go1.22.6
"Len64": true, // go1.22.6
"Len8": true, // go1.22.6
"OnesCount64": true, // go1.22.6
"Prefetch": true, // go1.22.6
"PrefetchStreamed": true, // go1.22.6
"TrailingZeros32": true, // go1.22.6
"TrailingZeros64": true, // go1.22.6
"TrailingZeros8": true, // go1.22.6
},
"sync": {
"runtime_LoadAcquintptr": true,
"runtime_StoreReluintptr": true,
"runtime_LoadAcquintptr": true, // go1.22.6
"runtime_StoreReluintptr": true, // go1.22.6
},
"sync/atomic": {
"AddInt32": true,
"AddInt64": true,
"AddUint32": true,
"AddUint64": true,
"AddUintptr": true,
"CompareAndSwapInt32": true,
"CompareAndSwapInt64": true,
"CompareAndSwapUint32": true,
"CompareAndSwapUint64": true,
"CompareAndSwapUintptr": true,
"LoadInt32": true,
"LoadInt64": true,
"LoadPointer": true,
"LoadUint32": true,
"LoadUint64": true,
"LoadUintptr": true,
"StoreInt32": true,
"StoreInt64": true,
"StoreUint32": true,
"StoreUint64": true,
"StoreUintptr": true,
"SwapInt32": true,
"SwapInt64": true,
"SwapUint32": true,
"SwapUint64": true,
"SwapUintptr": true,
"AddInt32": true, // go1.22.6
"AddInt64": true, // go1.22.6
"AddUint32": true, // go1.22.6
"AddUint64": true, // go1.22.6
"AddUintptr": true, // go1.22.6
"CompareAndSwapInt32": true, // go1.22.6
"CompareAndSwapInt64": true, // go1.22.6
"CompareAndSwapUint32": true, // go1.22.6
"CompareAndSwapUint64": true, // go1.22.6
"CompareAndSwapUintptr": true, // go1.22.6
"LoadInt32": true, // go1.22.6
"LoadInt64": true, // go1.22.6
"LoadPointer": true, // go1.22.6
"LoadUint32": true, // go1.22.6
"LoadUint64": true, // go1.22.6
"LoadUintptr": true, // go1.22.6
"StoreInt32": true, // go1.22.6
"StoreInt64": true, // go1.22.6
"StoreUint32": true, // go1.22.6
"StoreUint64": true, // go1.22.6
"StoreUintptr": true, // go1.22.6
"SwapInt32": true, // go1.22.6
"SwapInt64": true, // go1.22.6
"SwapUint32": true, // go1.22.6
"SwapUint64": true, // go1.22.6
"SwapUintptr": true, // go1.22.6
},
}

167
scripts/gen_go_std_tables.go
Unescape Escape View File

@ -20,22 +20,24 @@ import (
"text/template"
)
var goVersions = []string{"go1.22.6"}
var tmplTables = template.Must(template.New("").Parse(`
// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
// Generated from Go version {{ .GoVersion }}.
// Generated from Go versions {{ .GoVersions }}.
package main
var runtimeAndDeps = map[string]bool{
{{- range $path := .RuntimeAndDeps }}
"{{ $path }}": true,
"{{ $path.String }}": true, // {{ $path.GoVersion }}
{{- end }}
}
var runtimeLinknamed = []string{
{{- range $path := .RuntimeLinknamed }}
"{{ $path }}",
"{{ $path.String }}", // {{ $path.GoVersion }}
{{- end }}
// The net package linknames to the runtime, not the other way around.
// TODO: support this automatically via our script.
@ -46,7 +48,7 @@ var compilerIntrinsics = map[string]map[string]bool{
{{- range $intr := .CompilerIntrinsics }}
"{{ $intr.Path }}": {
{{- range $name := $intr.Names }}
"{{ $name }}": true,
"{{ $name.String }}": true, // {{ $name.GoVersion }}
{{- end }}
},
{{- end }}
@ -58,15 +60,15 @@ var reflectSkipPkg = map[string]bool{
`[1:]))
type tmplData struct {
GoVersion string
RuntimeAndDeps []string
RuntimeLinknamed []string
GoVersions []string
RuntimeAndDeps []versionedString
RuntimeLinknamed []versionedString
CompilerIntrinsics []tmplIntrinsic
}
type tmplIntrinsic struct {
Path string
Names []string
Names []versionedString
}
func (t tmplIntrinsic) Compare(t2 tmplIntrinsic) int {
@ -77,13 +79,36 @@ func (t tmplIntrinsic) Equal(t2 tmplIntrinsic) bool {
return t.Compare(t2) == 0
}
func cmdGo(args ...string) string {
type versionedString struct {
String string
GoVersion string
}
func (v versionedString) Compare(v2 versionedString) int {
if c := cmp.Compare(v.String, v2.String); c != 0 {
return c
}
// Negated so that newer Go versions go first.
return -cmp.Compare(v.GoVersion, v2.GoVersion)
}
func (v versionedString) Equal(v2 versionedString) bool {
// Note that we do equality based on String alone,
// because we only need one String entry with the latest version.
return v.String == v2.String
}
func cmdGo(goVersion string, args ...string) versionedString {
cmd := exec.Command("go", args...)
cmd.Env = append(cmd.Environ(), "GOTOOLCHAIN="+goVersion)
out, err := cmd.Output()
if err != nil {
panic(err)
}
return string(bytes.TrimSpace(out)) // no trailing newline
return versionedString{
String: string(bytes.TrimSpace(out)), // no trailing newline
GoVersion: goVersion,
}
}
func readFile(path string) string {
@ -94,81 +119,109 @@ func readFile(path string) string {
return string(data)
}
func sortedLines(s string) []string {
lines := strings.Split(s, "\n")
slices.Sort(lines)
lines = slices.Compact(lines)
return lines
func lines(vs versionedString) []versionedString {
split := strings.Split(vs.String, "\n")
var versioned []versionedString
for _, s := range split {
versioned = append(versioned, versionedString{
String: s,
GoVersion: vs.GoVersion,
})
}
return versioned
}
var rxLinkname = regexp.MustCompile(`^//go:linkname .* ([^.]*)\.[^.]*$`)
var rxIntrinsic = regexp.MustCompile(`\b(addF|alias)\("([^"]*)", "([^"]*)",`)
func main() {
goversion := cmdGo("env", "GOVERSION") // not "go version", to exclude GOOS/GOARCH
goroot := cmdGo("env", "GOROOT")
var runtimeAndDeps []versionedString
for _, goVersion := range goVersions {
runtimeAndDeps = append(runtimeAndDeps, lines(cmdGo(goVersion, "list", "-deps", "runtime"))...)
}
slices.SortFunc(runtimeAndDeps, versionedString.Compare)
runtimeAndDeps = slices.CompactFunc(runtimeAndDeps, versionedString.Equal)
runtimeAndDeps := sortedLines(cmdGo("list", "-deps", "runtime"))
var goroots []versionedString
for _, goVersion := range goVersions {
goroots = append(goroots, cmdGo(goVersion, "env", "GOROOT"))
}
// All packages that the runtime linknames to, except runtime and its dependencies.
// This resulting list is what we need to "go list" when obfuscating the runtime,
// as they are the packages that we may be missing.
var runtimeLinknamed []string
runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot, "src", "runtime", "*.go"))
if err != nil {
panic(err)
}
for _, goFile := range runtimeGoFiles {
for _, line := range strings.Split(readFile(goFile), "\n") {
m := rxLinkname.FindStringSubmatch(line)
if m == nil {
continue
}
path := m[1]
switch path {
case "main", "runtime/metrics_test":
continue
var runtimeLinknamed []versionedString
for _, goroot := range goroots {
runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot.String, "src", "runtime", "*.go"))
if err != nil {
panic(err)
}
for _, goFile := range runtimeGoFiles {
for _, line := range strings.Split(readFile(goFile), "\n") {
m := rxLinkname.FindStringSubmatch(line)
if m == nil {
continue
}
path := m[1]
switch path {
case "main", "runtime/metrics_test":
continue
}
runtimeLinknamed = append(runtimeLinknamed, versionedString{
String: path,
GoVersion: goroot.GoVersion,
})
}
runtimeLinknamed = append(runtimeLinknamed, path)
}
}
slices.Sort(runtimeLinknamed)
runtimeLinknamed = slices.Compact(runtimeLinknamed)
runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path string) bool {
return slices.Contains(runtimeAndDeps, path)
slices.SortFunc(runtimeLinknamed, versionedString.Compare)
runtimeLinknamed = slices.CompactFunc(runtimeLinknamed, versionedString.Equal)
runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path versionedString) bool {
for _, prev := range runtimeAndDeps {
if prev.String == path.String {
return true
}
}
return false
})
compilerIntrinsicsIndexByPath := make(map[string]int)
var compilerIntrinsics []tmplIntrinsic
for _, line := range strings.Split(readFile(filepath.Join(
goroot, "src", "cmd", "compile", "internal", "ssagen", "ssa.go",
)), "\n") {
m := rxIntrinsic.FindStringSubmatch(line)
if m == nil {
continue
}
path, name := m[2], m[3]
if i := compilerIntrinsicsIndexByPath[path]; i == 0 {
compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics)
compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{
Path: path,
Names: []string{name},
})
} else {
compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, name)
for _, goroot := range goroots {
for _, line := range strings.Split(readFile(filepath.Join(
goroot.String, "src", "cmd", "compile", "internal", "ssagen", "ssa.go",
)), "\n") {
m := rxIntrinsic.FindStringSubmatch(line)
if m == nil {
continue
}
path, name := m[2], m[3]
vs := versionedString{
String: name,
GoVersion: goroot.GoVersion,
}
if i := compilerIntrinsicsIndexByPath[path]; i == 0 {
compilerIntrinsicsIndexByPath[path] = len(compilerIntrinsics)
compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{
Path: path,
Names: []versionedString{vs},
})
} else {
compilerIntrinsics[i].Names = append(compilerIntrinsics[i].Names, vs)
}
}
}
slices.SortFunc(compilerIntrinsics, tmplIntrinsic.Compare)
compilerIntrinsics = slices.CompactFunc(compilerIntrinsics, tmplIntrinsic.Equal)
for path := range compilerIntrinsics {
intr := &compilerIntrinsics[path]
slices.Sort(intr.Names)
intr.Names = slices.Compact(intr.Names)
slices.SortFunc(intr.Names, versionedString.Compare)
intr.Names = slices.CompactFunc(intr.Names, versionedString.Equal)
}
var buf bytes.Buffer
if err := tmplTables.Execute(&buf, tmplData{
GoVersion: goversion,
GoVersions: goVersions,
RuntimeAndDeps: runtimeAndDeps,
RuntimeLinknamed: runtimeLinknamed,
CompilerIntrinsics: compilerIntrinsics,

Write Preview
Loading…
Cancel
Save