From cf290b8e6d593d07fa6f64ce1afcc4b9eee77145 Mon Sep 17 00:00:00 2001 From: lu4p Date: Mon, 9 Nov 2020 18:48:03 +0100 Subject: [PATCH] Share data between processes via a shared file. (#192) Previously garble heavily used env vars to share data between processes. This also makes it easy to share complex data between processes. The complexity of main.go is considerably reduced. --- hash.go | 10 +- import_obfuscation.go | 12 +- line_obfuscator.go | 2 +- main.go | 247 ++++++++----------------------------- shared.go | 219 ++++++++++++++++++++++++++++++++ testdata/scripts/basic.txt | 4 +- testdata/scripts/help.txt | 3 +- 7 files changed, 284 insertions(+), 213 deletions(-) create mode 100644 shared.go diff --git a/hash.go b/hash.go index 327db0a..db5a86b 100644 --- a/hash.go +++ b/hash.go @@ -111,14 +111,14 @@ func ownContentID(toolID []byte) (string, error) { if envGoPrivate != "" { fmt.Fprintf(h, " GOPRIVATE=%s", envGoPrivate) } - if envGarbleLiterals { + if opts.GarbleLiterals { fmt.Fprintf(h, " -literals") } - if envGarbleTiny { + if opts.Tiny { fmt.Fprintf(h, " -tiny") } - if envGarbleSeed != "" { - fmt.Fprintf(h, " -seed=%x", envGarbleSeed) + if len(opts.Seed) > 0 { + fmt.Fprintf(h, " -seed=%x", opts.Seed) } return hashToString(h.Sum(nil)), nil @@ -147,7 +147,7 @@ func hashWith(salt []byte, name string) string { d := sha256.New() d.Write(salt) - d.Write(seed) + d.Write(opts.Seed) io.WriteString(d, name) sum := b64.EncodeToString(d.Sum(nil)) diff --git a/import_obfuscation.go b/import_obfuscation.go index 351b19d..72127c1 100644 --- a/import_obfuscation.go +++ b/import_obfuscation.go @@ -77,7 +77,7 @@ func appendPrivateNameMap(pkg *goobj2.Package, nameMap map[string]string) error // extractDebugObfSrc extracts obfuscated sources from object files if -debugdir flag is enabled. func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error { - if envGarbleDebugDir == "" { + if opts.DebugDir == "" { return nil } @@ -94,7 +94,7 @@ func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error { } osPkgPath := filepath.FromSlash(pkgPath) - pkgDebugDir := filepath.Join(envGarbleDebugDir, osPkgPath) + pkgDebugDir := filepath.Join(opts.DebugDir, osPkgPath) if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil { return err } @@ -172,7 +172,7 @@ func obfuscateImports(objPath, tempDir string, importMap goobj2.ImportMap) (garb for pkgPath, info := range importCfg.Packages { // if the '-tiny' flag is passed, we will strip filename // and position info of every package, but not garble anything - if private := isPrivate(pkgPath); envGarbleTiny || private { + if private := isPrivate(pkgPath); opts.Tiny || private { pkg, err := goobj2.Parse(info.Path, pkgPath, importMap) if err != nil { return "", nil, nil, fmt.Errorf("error parsing objfile %s at %s: %v", pkgPath, info.Path, err) @@ -506,12 +506,12 @@ func garbleSymbols(am *goobj2.ArchiveMember, privImports privateImports, garbled } for _, inl := range s.Func.InlTree { inl.Func.Name = garbleSymbolName(inl.Func.Name, privImports, garbledImports, sb) - if envGarbleTiny { + if opts.Tiny { inl.Line = 1 } } - if envGarbleTiny { + if opts.Tiny { s.Func.PCFile = nil s.Func.PCLine = nil s.Func.PCInline = nil @@ -545,7 +545,7 @@ func garbleSymbolName(symName string, privImports privateImports, garbledImports // remove filename symbols when -tiny is passed // as they are only used for printing panics, // and -tiny removes panic printing - if envGarbleTiny && prefix == "gofile.." { + if opts.Tiny && prefix == "gofile.." { return prefix } diff --git a/line_obfuscator.go b/line_obfuscator.go index 1fe0b3b..01ce0eb 100644 --- a/line_obfuscator.go +++ b/line_obfuscator.go @@ -160,7 +160,7 @@ func transformLineInfo(file *ast.File, cgoFile bool) (detachedComments, localNam clearNodeComments(node) // If tiny mode is active information about line numbers is erased in object files - if envGarbleTiny { + if opts.Tiny { return true } funcDecl, ok := node.(*ast.FuncDecl) diff --git a/main.go b/main.go index cff1d40..26a69cb 100644 --- a/main.go +++ b/main.go @@ -7,10 +7,8 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/rand" "encoding/base64" "encoding/binary" - "encoding/gob" "encoding/json" "flag" "fmt" @@ -118,16 +116,9 @@ var ( return os.Open(buildInfo.imports[path].packagefile) }).(types.ImporterFrom) - envGoPrivate = os.Getenv("GOPRIVATE") // complemented by 'go env' later - - envGarbleDir = os.Getenv("GARBLE_DIR") - envGarbleLiterals = os.Getenv("GARBLE_LITERALS") == "true" - envGarbleTiny = os.Getenv("GARBLE_TINY") == "true" - envGarbleDebugDir = os.Getenv("GARBLE_DEBUGDIR") - envGarbleSeed = os.Getenv("GARBLE_SEED") - envGarbleListPkgs = os.Getenv("GARBLE_LISTPKGS") + opts *options - seed []byte + envGoPrivate = os.Getenv("GOPRIVATE") // complemented by 'go env' later ) const ( @@ -135,84 +126,6 @@ const ( garbleSrcHeaderName = "garble/src" ) -func saveListedPackages(w io.Writer, flags, patterns []string) error { - args := []string{"list", "-json", "-deps", "-export"} - args = append(args, flags...) - args = append(args, patterns...) - cmd := exec.Command("go", args...) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - - var stderr bytes.Buffer - cmd.Stderr = &stderr - - if err := cmd.Start(); err != nil { - return fmt.Errorf("go list error: %v", err) - } - dec := json.NewDecoder(stdout) - listedPackages = make(map[string]*listedPackage) - for dec.More() { - var pkg listedPackage - if err := dec.Decode(&pkg); err != nil { - return err - } - listedPackages[pkg.ImportPath] = &pkg - } - - if err := cmd.Wait(); err != nil { - return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes()) - } - if err := gob.NewEncoder(w).Encode(listedPackages); err != nil { - return err - } - return nil -} - -// listedPackages contains data obtained via 'go list -json -export -deps'. This -// allows us to obtain the non-garbled export data of all dependencies, useful -// for type checking of the packages as we obfuscate them. -// -// Note that we obtain this data once in saveListedPackages, store it into a -// temporary file via gob encoding, and then reuse that file in each of the -// garble processes that wrap a package compilation. -var listedPackages map[string]*listedPackage - -type listedPackage struct { - ImportPath string - Export string - Deps []string - ImportMap map[string]string - - // TODO(mvdan): reuse this field once TOOLEXEC_IMPORTPATH is used - private bool -} - -func listPackage(path string) (*listedPackage, error) { - if listedPackages == nil { - f, err := os.Open(envGarbleListPkgs) - if err != nil { - return nil, err - } - defer f.Close() - if err := gob.NewDecoder(f).Decode(&listedPackages); err != nil { - return nil, err - } - } - pkg, ok := listedPackages[path] - if !ok { - if fromPkg, ok := listedPackages[curPkgPath]; ok { - if path2 := fromPkg.ImportMap[path]; path2 != "" { - return listPackage(path2) - } - } - return nil, fmt.Errorf("path not found in listed packages: %s", path) - } - return pkg, nil -} - func garbledImport(path string) (*types.Package, error) { ipkg, ok := buildInfo.imports[path] if !ok { @@ -221,10 +134,10 @@ func garbledImport(path string) (*types.Package, error) { if ipkg.pkg != nil { return ipkg.pkg, nil // cached } - if envGarbleDir == "" { + if opts.GarbleDir == "" { return nil, fmt.Errorf("$GARBLE_DIR unset; did you run via 'garble build'?") } - pkg, err := garbledImporter.ImportFrom(path, envGarbleDir, 0) + pkg, err := garbledImporter.ImportFrom(path, opts.GarbleDir, 0) if err != nil { return nil, err } @@ -332,81 +245,11 @@ func mainErr(args []string) error { flagSet.Usage() } } - wd, err := os.Getwd() - if err != nil { - return err - } - os.Setenv("GARBLE_DIR", wd) - os.Setenv("GARBLE_LITERALS", fmt.Sprint(flagGarbleLiterals)) - os.Setenv("GARBLE_TINY", fmt.Sprint(flagGarbleTiny)) - - if flagSeed == "random" { - seed = make([]byte, 16) // random 128 bit seed - - if _, err := rand.Read(seed); err != nil { - return fmt.Errorf("error generating random seed: %v", err) - } - - flagSeed = "random;" + base64.StdEncoding.EncodeToString(seed) - } else { - flagSeed = strings.TrimRight(flagSeed, "=") - seed, err := base64.RawStdEncoding.DecodeString(flagSeed) - if err != nil { - return fmt.Errorf("error decoding seed: %v", err) - } - - if len(seed) != 0 && len(seed) < 8 { - return fmt.Errorf("the seed needs to be at least 8 bytes, but is only %v bytes", len(seed)) - } - - flagSeed = base64.StdEncoding.EncodeToString(seed) - } - - os.Setenv("GARBLE_SEED", flagSeed) - - if flagDebugDir != "" { - if !filepath.IsAbs(flagDebugDir) { - flagDebugDir = filepath.Join(wd, flagDebugDir) - } - - if err := os.RemoveAll(flagDebugDir); err == nil || os.IsNotExist(err) { - err := os.MkdirAll(flagDebugDir, 0o755) - if err != nil { - return err - } - } else { - return fmt.Errorf("debugdir error: %v", err) - } - } - - os.Setenv("GARBLE_DEBUGDIR", flagDebugDir) - if envGoPrivate == "" { - // Try 'go env' too, to query ${CONFIG}/go/env as well. - out, err := exec.Command("go", "env", "GOPRIVATE").CombinedOutput() - if err != nil { - return fmt.Errorf("%v: %s", err, out) - } - envGoPrivate = string(bytes.TrimSpace(out)) - } - // If GOPRIVATE isn't set and we're in a module, use its module - // path as a GOPRIVATE default. Include a _test variant too. - if envGoPrivate == "" { - modpath, err := exec.Command("go", "list", "-m").Output() - if err == nil { - path := string(bytes.TrimSpace(modpath)) - envGoPrivate = path + "," + path + "_test" - } - } - // Explicitly set GOPRIVATE, since future garble processes won't - // query 'go env' again. - os.Setenv("GOPRIVATE", envGoPrivate) - - f, err := ioutil.TempFile("", "garble-list-deps") + err := setOptions() if err != nil { return err } - defer os.Remove(f.Name()) // Note that we also need to pass build flags to 'go list', such // as -tags. @@ -414,34 +257,20 @@ func mainErr(args []string) error { if cmd == "test" { listFlags = append(listFlags, "-test") } - if err := saveListedPackages(f, listFlags, args); err != nil { + + if err := setGoPrivate(); err != nil { return err } - os.Setenv("GARBLE_LISTPKGS", f.Name()) - if err := f.Close(); err != nil { + + if err := setListedPackages(listFlags, args); err != nil { return err } - anyPrivate := false - for path, pkg := range listedPackages { - if isPrivate(path) { - pkg.private = true - anyPrivate = true - } - } - if !anyPrivate { - return fmt.Errorf("GOPRIVATE=%q does not match any packages to be built", envGoPrivate) - } - for path, pkg := range listedPackages { - if pkg.private { - continue - } - for _, depPath := range pkg.Deps { - if listedPackages[depPath].private { - return fmt.Errorf("public package %q can't depend on obfuscated package %q (matched via GOPRIVATE=%q)", - path, depPath, envGoPrivate) - } - } + + sharedName, err := saveShared() + if err != nil { + return err } + defer os.Remove(sharedName) execPath, err := os.Executable() if err != nil { @@ -471,6 +300,11 @@ func mainErr(args []string) error { return fmt.Errorf("unknown command: %q", args[0]) } + if err := loadShared(); err != nil { + return err + } + opts = &cache.Options + _, tool := filepath.Split(args[0]) if runtime.GOOS == "windows" { tool = strings.TrimSuffix(tool, ".exe") @@ -518,12 +352,12 @@ func transformCompile(args []string) ([]string, error) { flags = append(flags, "-dwarf=false") curPkgPath = flagValue(flags, "-p") - if (curPkgPath == "runtime" && envGarbleTiny) || curPkgPath == "runtime/internal/sys" { + if (curPkgPath == "runtime" && opts.Tiny) || curPkgPath == "runtime/internal/sys" { // Even though these packages aren't private, we will still process // them later to remove build information and strip code from the // runtime. However, we only want flags to work on private packages. - envGarbleLiterals = false - envGarbleDebugDir = "" + opts.GarbleLiterals = false + opts.DebugDir = "" } else if !isPrivate(curPkgPath) { return append(flags, paths...), nil } @@ -558,13 +392,8 @@ func transformCompile(args []string) ([]string, error) { files = append(files, file) } - if envGarbleSeed != "" { - seed, err = base64.StdEncoding.DecodeString(strings.TrimPrefix(envGarbleSeed, "random;")) - if err != nil { - return nil, fmt.Errorf("error decoding base64 seed: %v", err) - } - - mathrand.Seed(int64(binary.BigEndian.Uint64(seed))) + if len(opts.Seed) > 0 { + mathrand.Seed(int64(binary.BigEndian.Uint64(opts.Seed))) } else { mathrand.Seed(int64(binary.BigEndian.Uint64([]byte(curActionID)))) } @@ -587,7 +416,7 @@ func transformCompile(args []string) ([]string, error) { tf.existingNames = collectNames(files) tf.buildBlacklist(files) - if envGarbleLiterals { + if opts.GarbleLiterals { // TODO: use transformer here? files = literals.Obfuscate(files, tf.info, fset, tf.blacklist) } @@ -859,7 +688,7 @@ func (tf *transformer) buildBlacklist(files []*ast.File) { } visit := func(node ast.Node) bool { - if envGarbleLiterals { + if opts.GarbleLiterals { // TODO: use transformer here? literals.ConstBlacklist(node, tf.info, tf.blacklist) } @@ -1387,3 +1216,27 @@ func flagSetValue(flags []string, name, value string) []string { } return append(flags, name+"="+value) } + +func setGoPrivate() error { + if envGoPrivate == "" { + // Try 'go env' too, to query ${CONFIG}/go/env as well. + out, err := exec.Command("go", "env", "GOPRIVATE").CombinedOutput() + if err != nil { + return fmt.Errorf("%v: %s", err, out) + } + envGoPrivate = string(bytes.TrimSpace(out)) + } + // If GOPRIVATE isn't set and we're in a module, use its module + // path as a GOPRIVATE default. Include a _test variant too. + if envGoPrivate == "" { + modpath, err := exec.Command("go", "list", "-m").Output() + if err == nil { + path := string(bytes.TrimSpace(modpath)) + envGoPrivate = path + "," + path + "_test" + } + } + // Explicitly set GOPRIVATE, since future garble processes won't + // query 'go env' again. + os.Setenv("GOPRIVATE", envGoPrivate) + return nil +} diff --git a/shared.go b/shared.go new file mode 100644 index 0000000..a33f3f2 --- /dev/null +++ b/shared.go @@ -0,0 +1,219 @@ +package main + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/gob" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// shared this data is shared between the different garble processes +type shared struct { + Options options + ListedPackages listedPackages +} + +var cache *shared + +// loadShared the shared data passed from the entry garble process +func loadShared() error { + if cache == nil { + f, err := os.Open(os.Getenv("GARBLE_SHARED")) + if err != nil { + return fmt.Errorf(`cannot open shared file, this is most likely due to not running "garble [command]"`) + } + defer f.Close() + if err := gob.NewDecoder(f).Decode(&cache); err != nil { + return err + } + } + + return nil +} + +// saveShared the shared data to a file in order for subsequent +// garble processes to have access to the same data +func saveShared() (string, error) { + f, err := ioutil.TempFile("", "garble-shared") + if err != nil { + return "", err + } + + defer f.Close() + + if err := gob.NewEncoder(f).Encode(&cache); err != nil { + return "", err + } + + os.Setenv("GARBLE_SHARED", f.Name()) + + return f.Name(), nil +} + +// options are derived from the flags +type options struct { + GarbleLiterals bool + Tiny bool + GarbleDir string + DebugDir string + Seed []byte + Random bool +} + +// setOptions sets all options from the user supplied flags +func setOptions() error { + wd, err := os.Getwd() + if err != nil { + return err + } + + opts = &options{ + GarbleDir: wd, + GarbleLiterals: flagGarbleLiterals, + Tiny: flagGarbleTiny, + } + + if flagSeed == "random" { + opts.Seed = make([]byte, 16) // random 128 bit seed + if _, err := rand.Read(opts.Seed); err != nil { + return fmt.Errorf("error generating random seed: %v", err) + } + + opts.Random = true + + } else { + flagSeed = strings.TrimRight(flagSeed, "=") + seed, err := base64.RawStdEncoding.DecodeString(flagSeed) + if err != nil { + return fmt.Errorf("error decoding seed: %v", err) + } + + if len(seed) != 0 && len(seed) < 8 { + return fmt.Errorf("the seed needs to be at least 8 bytes, but is only %v bytes", len(seed)) + } + + opts.Seed = seed + } + + if flagDebugDir != "" { + if !filepath.IsAbs(flagDebugDir) { + flagDebugDir = filepath.Join(wd, flagDebugDir) + } + + if err := os.RemoveAll(flagDebugDir); err == nil || os.IsNotExist(err) { + err := os.MkdirAll(flagDebugDir, 0o755) + if err != nil { + return err + } + } else { + return fmt.Errorf("debugdir error: %v", err) + } + + opts.DebugDir = flagDebugDir + } + + cache = &shared{Options: *opts} + + return nil +} + +// listedPackages contains data obtained via 'go list -json -export -deps'. This +// allows us to obtain the non-garbled export data of all dependencies, useful +// for type checking of the packages as we obfuscate them. +// +// Note that we obtain this data once in saveListedPackages, store it into a +// temporary file via gob encoding, and then reuse that file in each of the +// garble processes that wrap a package compilation. +type listedPackages map[string]*listedPackage + +// listedPackage contains information useful for obfuscating a package +type listedPackage struct { + ImportPath string + Export string + Deps []string + ImportMap map[string]string + + // TODO(mvdan): reuse this field once TOOLEXEC_IMPORTPATH is used + private bool +} + +// setListedPackages gets information about the current package +// and all of its dependencies +func setListedPackages(flags, patterns []string) error { + args := []string{"list", "-json", "-deps", "-export"} + args = append(args, flags...) + args = append(args, patterns...) + cmd := exec.Command("go", args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + return fmt.Errorf("go list error: %v", err) + } + dec := json.NewDecoder(stdout) + cache.ListedPackages = make(listedPackages) + for dec.More() { + var pkg listedPackage + if err := dec.Decode(&pkg); err != nil { + return err + } + cache.ListedPackages[pkg.ImportPath] = &pkg + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes()) + } + + anyPrivate := false + for path, pkg := range cache.ListedPackages { + if isPrivate(path) { + pkg.private = true + anyPrivate = true + } + } + + if !anyPrivate { + return fmt.Errorf("GOPRIVATE=%q does not match any packages to be built", os.Getenv("GOPRIVATE")) + } + for path, pkg := range cache.ListedPackages { + if pkg.private { + continue + } + for _, depPath := range pkg.Deps { + if cache.ListedPackages[depPath].private { + return fmt.Errorf("public package %q can't depend on obfuscated package %q (matched via GOPRIVATE=%q)", + path, depPath, os.Getenv("GOPRIVATE")) + } + } + } + + return nil + +} + +// listPackage gets the listedPackage information for a certain package +func listPackage(path string) (*listedPackage, error) { + pkg, ok := cache.ListedPackages[path] + if !ok { + if fromPkg, ok := cache.ListedPackages[curPkgPath]; ok { + if path2 := fromPkg.ImportMap[path]; path2 != "" { + return listPackage(path2) + } + } + return nil, fmt.Errorf("path not found in listed packages: %s", path) + } + return pkg, nil +} diff --git a/testdata/scripts/basic.txt b/testdata/scripts/basic.txt index f6b6562..3b42789 100644 --- a/testdata/scripts/basic.txt +++ b/testdata/scripts/basic.txt @@ -27,9 +27,9 @@ stdout 'unknown' [short] stop # checking that the build is reproducible is slow -# Check that we fail if the user ran with -toolexec but without -trimpath. +# Check that we fail if the user used "go build -toolexec garble" instead of "garble build" ! exec go build -a -toolexec=garble main.go -stderr 'should be used alongside -trimpath' +stderr 'not running "garble \[command\]"' # Also check that the binary is reproducible. # No packages should be rebuilt either, thanks to the build cache. diff --git a/testdata/scripts/help.txt b/testdata/scripts/help.txt index efbbc15..97bb306 100644 --- a/testdata/scripts/help.txt +++ b/testdata/scripts/help.txt @@ -25,6 +25,5 @@ stderr 'garble \[flags\] build' stderr 'unknown command' [!windows] ! garble /does/not/exist/compile -[!windows] stderr 'no such file' [windows] ! garble C:\does\not\exist\compile -[windows] stderr 'file does not exist' +stderr 'not running "garble \[command\]"'