From 2259abb89f7b96fa51482e75b7572a9ecd5f8592 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Mart=C3=AD?= <mvdan@mvdan.cc>
Date: Mon, 26 Aug 2024 23:51:28 +0100
Subject: [PATCH] rewrite generation of go_std_tables.go from Bash to Go

This makes it portable and easier to maintain for any Go developers.
I also want to improve its logic, which would have been harder in shell.
---
 go_std_tables.go             |  40 ++++----
 scripts/gen-go-std-tables.sh |  63 ------------
 scripts/gen_go_std_tables.go | 182 +++++++++++++++++++++++++++++++++++
 shared.go                    |   2 +-
 4 files changed, 203 insertions(+), 84 deletions(-)
 delete mode 100755 scripts/gen-go-std-tables.sh
 create mode 100644 scripts/gen_go_std_tables.go

diff --git a/go_std_tables.go b/go_std_tables.go
index 01e22df..d5cabbc 100644
--- a/go_std_tables.go
+++ b/go_std_tables.go
@@ -1,25 +1,25 @@
-// Code generated by scripts/gen-go-std-tables.sh; DO NOT EDIT.
+// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
 
 // Generated from Go version go1.22.0.
 
 package main
 
 var runtimeAndDeps = map[string]bool{
-	"internal/goarch":          true,
-	"unsafe":                   true,
 	"internal/abi":             true,
-	"internal/cpu":             true,
 	"internal/bytealg":         true,
 	"internal/chacha8rand":     true,
 	"internal/coverage/rtcov":  true,
+	"internal/cpu":             true,
+	"internal/goarch":          true,
 	"internal/godebugs":        true,
 	"internal/goexperiment":    true,
 	"internal/goos":            true,
+	"runtime":                  true,
 	"runtime/internal/atomic":  true,
 	"runtime/internal/math":    true,
 	"runtime/internal/sys":     true,
 	"runtime/internal/syscall": true,
-	"runtime":                  true,
+	"unsafe":                   true,
 }
 
 var runtimeLinknamed = []string{
@@ -67,6 +67,14 @@ var compilerIntrinsicsPkgs = map[string]bool{
 
 var compilerIntrinsicsFuncs = map[string]bool{
 	"math.Abs":                                true,
+	"math.Ceil":                               true,
+	"math.Copysign":                           true,
+	"math.FMA":                                true,
+	"math.Floor":                              true,
+	"math.Round":                              true,
+	"math.RoundToEven":                        true,
+	"math.Trunc":                              true,
+	"math.sqrt":                               true,
 	"math/big.mulWW":                          true,
 	"math/bits.Add":                           true,
 	"math/bits.Add64":                         true,
@@ -103,22 +111,15 @@ var compilerIntrinsicsFuncs = map[string]bool{
 	"math/bits.TrailingZeros32":               true,
 	"math/bits.TrailingZeros64":               true,
 	"math/bits.TrailingZeros8":                true,
-	"math.Ceil":                               true,
-	"math.Copysign":                           true,
-	"math.Floor":                              true,
-	"math.FMA":                                true,
-	"math.Round":                              true,
-	"math.RoundToEven":                        true,
-	"math.sqrt":                               true,
-	"math.Trunc":                              true,
+	"runtime.publicationBarrier":              true,
 	"runtime/internal/atomic.And":             true,
 	"runtime/internal/atomic.And8":            true,
 	"runtime/internal/atomic.Cas":             true,
 	"runtime/internal/atomic.Cas64":           true,
+	"runtime/internal/atomic.CasRel":          true,
 	"runtime/internal/atomic.Casint32":        true,
 	"runtime/internal/atomic.Casint64":        true,
 	"runtime/internal/atomic.Casp1":           true,
-	"runtime/internal/atomic.CasRel":          true,
 	"runtime/internal/atomic.Casuintptr":      true,
 	"runtime/internal/atomic.Load":            true,
 	"runtime/internal/atomic.Load64":          true,
@@ -136,12 +137,12 @@ var compilerIntrinsicsFuncs = map[string]bool{
 	"runtime/internal/atomic.Store":           true,
 	"runtime/internal/atomic.Store64":         true,
 	"runtime/internal/atomic.Store8":          true,
-	"runtime/internal/atomic.Storeint32":      true,
-	"runtime/internal/atomic.Storeint64":      true,
-	"runtime/internal/atomic.StorepNoWB":      true,
 	"runtime/internal/atomic.StoreRel":        true,
 	"runtime/internal/atomic.StoreRel64":      true,
 	"runtime/internal/atomic.StoreReluintptr": true,
+	"runtime/internal/atomic.Storeint32":      true,
+	"runtime/internal/atomic.Storeint64":      true,
+	"runtime/internal/atomic.StorepNoWB":      true,
 	"runtime/internal/atomic.Storeuintptr":    true,
 	"runtime/internal/atomic.Xadd":            true,
 	"runtime/internal/atomic.Xadd64":          true,
@@ -166,7 +167,8 @@ var compilerIntrinsicsFuncs = map[string]bool{
 	"runtime/internal/sys.TrailingZeros32":    true,
 	"runtime/internal/sys.TrailingZeros64":    true,
 	"runtime/internal/sys.TrailingZeros8":     true,
-	"runtime.publicationBarrier":              true,
+	"sync.runtime_LoadAcquintptr":             true,
+	"sync.runtime_StoreReluintptr":            true,
 	"sync/atomic.AddInt32":                    true,
 	"sync/atomic.AddInt64":                    true,
 	"sync/atomic.AddUint32":                   true,
@@ -193,8 +195,6 @@ var compilerIntrinsicsFuncs = map[string]bool{
 	"sync/atomic.SwapUint32":                  true,
 	"sync/atomic.SwapUint64":                  true,
 	"sync/atomic.SwapUintptr":                 true,
-	"sync.runtime_LoadAcquintptr":             true,
-	"sync.runtime_StoreReluintptr":            true,
 }
 
 var reflectSkipPkg = map[string]bool{
diff --git a/scripts/gen-go-std-tables.sh b/scripts/gen-go-std-tables.sh
deleted file mode 100755
index 2ebe255..0000000
--- a/scripts/gen-go-std-tables.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-
-# We can rewrite this bash script in Go if a dependency on bash and coreutils
-# is a problem during development.
-
-goroot=$(go env GOROOT)
-go_version=$(go env GOVERSION) # not "go version", to exclude GOOS/GOARCH
-
-runtime_and_deps=$(go list -deps runtime)
-
-# All packages that the runtime linknames to, except runtime and its dependencies.
-# This resulting list is what we need to "go list" when obfuscating the runtime,
-# as they are the packages that we may be missing.
-runtime_linknamed=$(comm -23 <(
-	sed -rn 's@//go:linkname .* ([^.]*)\.[^.]*@\1@p' "${goroot}"/src/runtime/*.go | grep -vE '^main|^runtime\.|_test$' | sort -u
-) <(
-	# Note that we assume this is constant across platforms.
-	go list -deps runtime | sort -u
-))
-
-compiler_intrinsics_table="$(sed -rn 's@.*\b(addF|alias)\("([^"]*)", "([^"]*)",.*@\2 \3@p' "${goroot}"/src/cmd/compile/internal/ssagen/ssa.go | sort -u)"
-compiler_intrinsics_paths="$(while read path name; do
-	echo ${path}
-done <<<"${compiler_intrinsics_table}" | sort -u)"
-
-gofmt >go_std_tables.go <<EOF
-// Code generated by scripts/gen-go-std-tables.sh; DO NOT EDIT.
-
-// Generated from Go version ${go_version}.
-
-package main
-
-var runtimeAndDeps = map[string]bool{
-$(for path in ${runtime_and_deps}; do
-	echo "\"${path}\": true,"
-done)
-}
-
-var runtimeLinknamed = []string{
-$(for path in ${runtime_linknamed}; do
-	echo "\"${path}\"",
-done)
-	// The net package linknames to the runtime, not the other way around.
-	// TODO: support this automatically via our script.
-	"net",
-}
-
-var compilerIntrinsicsPkgs = map[string]bool{
-$(for path in ${compiler_intrinsics_paths}; do
-	echo "\"${path}\": true,"
-done)
-}
-
-var compilerIntrinsicsFuncs = map[string]bool{
-$(while read path name; do
-	echo "\"${path}.${name}\": true,"
-done <<<"${compiler_intrinsics_table}")
-}
-
-var reflectSkipPkg = map[string]bool{
-	"fmt": true,
-}
-EOF
diff --git a/scripts/gen_go_std_tables.go b/scripts/gen_go_std_tables.go
new file mode 100644
index 0000000..e31bc84
--- /dev/null
+++ b/scripts/gen_go_std_tables.go
@@ -0,0 +1,182 @@
+// Copyright (c) 2024, The Garble Authors.
+// See LICENSE for licensing information.
+
+// This is a program used with `go generate`, so it handles errors via panic.
+package main
+
+import (
+	"bytes"
+	"cmp"
+	"fmt"
+	"go/format"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+	"text/template"
+)
+
+var tmplTables = template.Must(template.New("").Parse(`
+// Code generated by scripts/gen_go_std_tables.go; DO NOT EDIT.
+
+// Generated from Go version {{ .GoVersion }}.
+
+package main
+
+var runtimeAndDeps = map[string]bool{
+{{- range $path := .RuntimeAndDeps }}
+	"{{ $path }}": true,
+{{- end }}
+}
+
+var runtimeLinknamed = []string{
+{{- range $path := .RuntimeLinknamed }}
+	"{{ $path }}",
+{{- end }}
+	// The net package linknames to the runtime, not the other way around.
+	// TODO: support this automatically via our script.
+	"net",
+}
+
+var compilerIntrinsicsPkgs = map[string]bool{
+{{- range $path := .CompilerIntrinsicsPaths }}
+	"{{ $path }}": true,
+{{- end }}
+}
+
+var compilerIntrinsicsFuncs = map[string]bool{
+{{- range $intr := .CompilerIntrinsics }}
+	"{{ $intr.Path }}.{{ $intr.Name }}": true,
+{{- end }}
+}
+
+var reflectSkipPkg = map[string]bool{
+	"fmt": true,
+}
+`))
+
+type tmplData struct {
+	GoVersion               string
+	RuntimeAndDeps          []string
+	RuntimeLinknamed        []string
+	CompilerIntrinsics      []tmplIntrinsic
+	CompilerIntrinsicsPaths []string
+}
+
+type tmplIntrinsic struct {
+	Path, Name string
+}
+
+func (t tmplIntrinsic) Compare(t2 tmplIntrinsic) int {
+	if c := cmp.Compare(t.Path, t2.Path); c != 0 {
+		return c
+	}
+	return cmp.Compare(t.Name, t2.Name)
+}
+
+func (t tmplIntrinsic) Equal(t2 tmplIntrinsic) bool {
+	return t.Compare(t2) == 0
+}
+
+func cmdGo(args ...string) string {
+	cmd := exec.Command("go", args...)
+	out, err := cmd.Output()
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes.TrimSpace(out)) // no trailing newline
+}
+
+func readFile(path string) string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		panic(err)
+	}
+	return string(data)
+}
+
+func sortedLines(s string) []string {
+	lines := strings.Split(s, "\n")
+	slices.Sort(lines)
+	lines = slices.Compact(lines)
+	return lines
+}
+
+var rxLinkname = regexp.MustCompile(`^//go:linkname .* ([^.]*)\.[^.]*$`)
+var rxIntrinsic = regexp.MustCompile(`\b(addF|alias)\("([^"]*)", "([^"]*)",`)
+
+func main() {
+	goversion := cmdGo("env", "GOVERSION") // not "go version", to exclude GOOS/GOARCH
+	goroot := cmdGo("env", "GOROOT")
+
+	runtimeAndDeps := sortedLines(cmdGo("list", "-deps", "runtime"))
+
+	// All packages that the runtime linknames to, except runtime and its dependencies.
+	// This resulting list is what we need to "go list" when obfuscating the runtime,
+	// as they are the packages that we may be missing.
+	var runtimeLinknamed []string
+	runtimeGoFiles, err := filepath.Glob(filepath.Join(goroot, "src", "runtime", "*.go"))
+	if err != nil {
+		panic(err)
+	}
+	for _, goFile := range runtimeGoFiles {
+		for _, line := range strings.Split(readFile(goFile), "\n") {
+			m := rxLinkname.FindStringSubmatch(line)
+			if m == nil {
+				continue
+			}
+			path := m[1]
+			switch path {
+			case "main", "runtime/metrics_test":
+				continue
+			}
+			runtimeLinknamed = append(runtimeLinknamed, path)
+		}
+	}
+	slices.Sort(runtimeLinknamed)
+	runtimeLinknamed = slices.Compact(runtimeLinknamed)
+	runtimeLinknamed = slices.DeleteFunc(runtimeLinknamed, func(path string) bool {
+		return slices.Contains(runtimeAndDeps, path)
+	})
+
+	var compilerIntrinsics []tmplIntrinsic
+	var compilerIntrinsicsPaths []string
+	for _, line := range strings.Split(readFile(filepath.Join(
+		goroot, "src", "cmd", "compile", "internal", "ssagen", "ssa.go",
+	)), "\n") {
+		m := rxIntrinsic.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		compilerIntrinsics = append(compilerIntrinsics, tmplIntrinsic{
+			Path: m[2],
+			Name: m[3],
+		})
+		compilerIntrinsicsPaths = append(compilerIntrinsicsPaths, m[2])
+	}
+	slices.SortFunc(compilerIntrinsics, tmplIntrinsic.Compare)
+	compilerIntrinsics = slices.CompactFunc(compilerIntrinsics, tmplIntrinsic.Equal)
+	slices.Sort(compilerIntrinsicsPaths)
+	compilerIntrinsicsPaths = slices.Compact(compilerIntrinsicsPaths)
+
+	var buf bytes.Buffer
+	tmplTables.Execute(&buf, tmplData{
+		GoVersion:               goversion,
+		RuntimeAndDeps:          runtimeAndDeps,
+		RuntimeLinknamed:        runtimeLinknamed,
+		CompilerIntrinsics:      compilerIntrinsics,
+		CompilerIntrinsicsPaths: compilerIntrinsicsPaths,
+	})
+	out := buf.Bytes()
+	formatted, err := format.Source(out)
+	if err != nil {
+		fmt.Println(string(out))
+		panic(err)
+	}
+
+	if err := os.WriteFile("go_std_tables.go", formatted, 0o666); err != nil {
+		panic(err)
+	}
+}
diff --git a/shared.go b/shared.go
index 1c3cb67..a77ee81 100644
--- a/shared.go
+++ b/shared.go
@@ -20,7 +20,7 @@ import (
 	"golang.org/x/mod/module"
 )
 
-//go:generate ./scripts/gen-go-std-tables.sh
+//go:generate go run scripts/gen_go_std_tables.go
 
 // sharedCacheType is shared as a read-only cache between the many garble toolexec
 // sub-processes.