From 65461aabce84f3f1e7ab9de47adce91977775ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Wed, 15 Jul 2020 21:37:48 +0100 Subject: [PATCH] reuse a single 'go list -json -export -deps' call Instead of doing a 'go list' call every time we need to fetch a dependency's export file, we now do a single 'go list' call before the build begins. With the '-deps' flag, it gives us all the dependency packages recursively. We store that data in the gob format in a temporary file, and share it with the future garble sub-processes via an env var. This required lazy parsing of flags for the 'build' and 'test' commands, since now we need to run 'go list' with the same package pattern arguments. Fixes #63. --- go.mod | 1 + go.sum | 2 + main.go | 155 ++++++++++++++++++++++++++++------ main_test.go | 44 ++++++++++ testdata/scripts/debugdir.txt | 2 +- 5 files changed, 179 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index 2e624bd..f253c1b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module mvdan.cc/garble go 1.14 require ( + github.com/google/go-cmp v0.5.1 github.com/rogpeppe/go-internal v1.6.0 golang.org/x/tools v0.0.0-20200622203043-20e05c1c8ffa ) diff --git a/go.sum b/go.sum index 67747cf..f56cf0c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/main.go b/main.go index 708c6e2..9a0e0bd 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/binary" + "encoding/gob" "encoding/json" "flag" "fmt" @@ -96,32 +97,78 @@ var ( envGarbleDebugDir = os.Getenv("GARBLE_DEBUGDIR") envGarbleSeed = os.Getenv("GARBLE_SEED") envGoPrivate string // filled via 'go env' below to support 'go env -w' + envGarbleListPkgs = os.Getenv("GARBLE_LISTPKGS") seed []byte ) +func saveListedPackages(w io.Writer, test bool, patterns ...string) error { + args := []string{"list", "-json", "-deps", "-export"} + if test { + args = append(args, "-test") + } + args = append(args, patterns...) + cmd := exec.Command("go", args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + return fmt.Errorf("go list error: %v", err) + } + dec := json.NewDecoder(stdout) + listedPackages = make(map[string]*listedPackage) + for dec.More() { + var pkg listedPackage + if err := dec.Decode(&pkg); err != nil { + return err + } + listedPackages[pkg.ImportPath] = &pkg + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes()) + } + if err := gob.NewEncoder(w).Encode(listedPackages); err != nil { + return err + } + return nil +} + +// listedPackages contains data obtained via 'go list -json -export -deps'. This +// allows us to obtain the non-garbled export data of all dependencies, useful +// for type checking of the packages as we obfuscate them. +// +// Note that we obtain this data once in saveListedPackages, store it into a +// temporary file via gob encoding, and then reuse that file in each of the +// garble processes that wrap a package compilation. +var listedPackages map[string]*listedPackage + type listedPackage struct { - Export string - Deps []string + ImportPath string + Export string + Deps []string } -// listPackage is a simple wrapper around 'go list -json'. -func listPackage(path string) (listedPackage, error) { - var pkg listedPackage - cmd := exec.Command("go", "list", "-json", "-export", path) - if envGarbleDir == "" { - return pkg, fmt.Errorf("$GARBLE_DIR unset; did you run via 'garble build'?") - } - cmd.Dir = envGarbleDir - out, err := cmd.Output() - if err != nil { - if err, _ := err.(*exec.ExitError); err != nil { - return pkg, fmt.Errorf("go list error: %v: %s", err, err.Stderr) +func listPackage(path string) (*listedPackage, error) { + if listedPackages == nil { + f, err := os.Open(envGarbleListPkgs) + if err != nil { + return nil, err + } + defer f.Close() + if err := gob.NewDecoder(f).Decode(&listedPackages); err != nil { + return nil, err } - return pkg, fmt.Errorf("go list error: %v", err) } - if err := json.Unmarshal(out, &pkg); err != nil { - return pkg, err + pkg, ok := listedPackages[path] + if !ok { + return nil, fmt.Errorf("path not found in listed packages: %s", path) } return pkg, nil } @@ -187,8 +234,11 @@ func mainErr(args []string) error { case "help": flagSet.Usage() case "build", "test": - if len(args) > 1 { - switch args[1] { + // Split the flags from the package arguments, since we'll need + // to run 'go list' on the same set of packages. + flags, args := splitFlagsFromArgs(args[1:]) + for _, flag := range flags { + switch flag { case "-h", "-help", "--help": flagSet.Usage() } @@ -241,6 +291,21 @@ func mainErr(args []string) error { } } + f, err := ioutil.TempFile("", "garble-list-deps") + if err != nil { + return err + } + defer os.Remove(f.Name()) + // TODO: Pass along flags that 'go list' understands too, such + // as -mod or -modfile. + if err := saveListedPackages(f, cmd == "test", args...); err != nil { + return err + } + os.Setenv("GARBLE_LISTPKGS", f.Name()) + if err := f.Close(); err != nil { + return err + } + execPath, err := os.Executable() if err != nil { return err @@ -256,7 +321,8 @@ func mainErr(args []string) error { // disabled by default. goArgs = append(goArgs, "-vet=off") } - goArgs = append(goArgs, args[1:]...) + goArgs = append(goArgs, flags...) + goArgs = append(goArgs, args...) cmd := exec.Command("go", goArgs...) cmd.Stdout = os.Stdout @@ -907,17 +973,58 @@ func transformLink(args []string) ([]string, error) { return append(flags, paths...), nil } +func splitFlagsFromArgs(all []string) (flags, args []string) { + for i := 0; i < len(all); i++ { + arg := all[i] + if !strings.HasPrefix(arg, "-") { + return all[:i], all[i:] + } + if booleanFlags[arg] || strings.Contains(arg, "=") { + // Either "-bool" or "-name=value". + continue + } + // "-name value", so the next arg is part of this flag. + i++ + } + return all, nil +} + +var booleanFlags = map[string]bool{ + // Shared build flags. + "-a": true, + "-i": true, + "-n": true, + "-v": true, + "-x": true, + "-race": true, + "-msan": true, + "-linkshared": true, + "-modcacherw": true, + "-trimpath": true, + + // Test flags (TODO: support its special -args flag) + "-c": true, + "-json": true, + "-cover": true, + "-failfast": true, + "-short": true, + "-benchmem": true, +} + // splitFlagsFromFiles splits args into a list of flag and file arguments. Since // we can't rely on "--" being present, and we don't parse all flags upfront, we // rely on finding the first argument that doesn't begin with "-" and that has // the extension we expect for the list of paths. -func splitFlagsFromFiles(args []string, ext string) (flags, paths []string) { - for i, arg := range args { +// +// This function only makes sense for lower-level tool commands, such as +// "compile" or "link", since their arguments are predictable. +func splitFlagsFromFiles(all []string, ext string) (flags, paths []string) { + for i, arg := range all { if !strings.HasPrefix(arg, "-") && strings.HasSuffix(arg, ext) { - return args[:i:i], args[i:] + return all[:i:i], all[i:] } } - return args, nil + return all, nil } // flagValue retrieves the value of a flag such as "-foo", from strings in the diff --git a/main_test.go b/main_test.go index 2b59f1c..c8b75e0 100644 --- a/main_test.go +++ b/main_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/rogpeppe/go-internal/goproxytest" "github.com/rogpeppe/go-internal/gotooltest" "github.com/rogpeppe/go-internal/testscript" @@ -325,6 +326,49 @@ func generateLiterals(ts *testscript.TestScript, neg bool, args []string) { } } +func TestSplitFlagsFromArgs(t *testing.T) { + t.Parallel() + tests := []struct { + name string + args []string + want [2][]string + }{ + {"Empty", []string{}, [2][]string{{}, nil}}, + { + "JustFlags", + []string{"-foo", "bar", "-baz"}, + [2][]string{{"-foo", "bar", "-baz"}, nil}, + }, + { + "JustArgs", + []string{"some", "pkgs"}, + [2][]string{{}, {"some", "pkgs"}}, + }, + { + "FlagsAndArgs", + []string{"-foo=bar", "baz"}, + [2][]string{{"-foo=bar"}, {"baz"}}, + }, + { + "BoolFlagsAndArgs", + []string{"-race", "pkg"}, + [2][]string{{"-race"}, {"pkg"}}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + flags, args := splitFlagsFromArgs(test.args) + got := [2][]string{flags, args} + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Fatalf("splitFlagsFromArgs(%q) mismatch (-want +got):\n%s", test.args, diff) + } + }) + } +} + func TestFlagValue(t *testing.T) { t.Parallel() tests := []struct { diff --git a/testdata/scripts/debugdir.txt b/testdata/scripts/debugdir.txt index 8e2d89f..ab15769 100644 --- a/testdata/scripts/debugdir.txt +++ b/testdata/scripts/debugdir.txt @@ -18,4 +18,4 @@ func main() { -- imported/imported.go -- package imported -func ImportedFunc() {} \ No newline at end of file +func ImportedFunc() {}