From d8e87382162d0eb684a8e9fd2e048d99c54ae4ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 16 Feb 2021 00:23:45 +0000 Subject: [PATCH] initial support for reversing panic output (#225) For now, this only implements reversing of exported names which are hashed with action IDs. Many other kinds of obfuscation, like positions and private names, are not yet implemented. Note that we don't document this new command yet on purpose, since it's not finished. Some other minor cleanups were done for future changes, such as making transformLineInfo into a method that also receives the original filename, and making header names more self-describing. Updates #5. --- import_obfuscation.go | 4 +- line_obfuscator.go | 11 ++- main.go | 144 ++++++++++++++++++--------------- reverse.go | 149 +++++++++++++++++++++++++++++++++++ shared.go | 3 +- testdata/scripts/reverse.txt | 67 ++++++++++++++++ 6 files changed, 308 insertions(+), 70 deletions(-) create mode 100644 reverse.go create mode 100644 testdata/scripts/reverse.txt diff --git a/import_obfuscation.go b/import_obfuscation.go index bcf2477..189232c 100644 --- a/import_obfuscation.go +++ b/import_obfuscation.go @@ -61,7 +61,7 @@ type privateName struct { func appendPrivateNameMap(pkg *goobj2.Package, nameMap map[string]string) error { for _, member := range pkg.ArchiveMembers { - if member.ArchiveHeader.Name != garbleMapHeaderName { + if member.ArchiveHeader.Name != headerPrivateNameMap { continue } @@ -83,7 +83,7 @@ func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error { var archiveMember *goobj2.ArchiveMember for _, member := range pkg.ArchiveMembers { - if member.ArchiveHeader.Name == garbleSrcHeaderName { + if member.ArchiveHeader.Name == headerDebugSource { archiveMember = &member break } diff --git a/line_obfuscator.go b/line_obfuscator.go index 77c24a3..6cf9e72 100644 --- a/line_obfuscator.go +++ b/line_obfuscator.go @@ -94,9 +94,9 @@ func clearNodeComments(node ast.Node) { // transformLineInfo removes the comment except go directives and build tags. Converts comments to the node view. // It returns comments not attached to declarations and names of declarations which cannot be renamed. -func transformLineInfo(file *ast.File, cgoFile bool) (detachedComments []string, f *ast.File) { +func (tf *transformer) transformLineInfo(file *ast.File, name string) (detachedComments []string, f *ast.File) { prefix := "" - if cgoFile { + if strings.HasPrefix(name, "_cgo_") { prefix = "_cgo_" } @@ -126,8 +126,13 @@ func transformLineInfo(file *ast.File, cgoFile bool) (detachedComments []string, if !ok { return true } + newPos := fmt.Sprintf("%s%c.go:%d", + prefix, + nameCharset[mathrand.Intn(len(nameCharset))], + PosMin+newLines[funcCounter], + ) - comment := &ast.Comment{Text: fmt.Sprintf("//line %s%c.go:%d", prefix, nameCharset[mathrand.Intn(len(nameCharset))], PosMin+newLines[funcCounter])} + comment := &ast.Comment{Text: "//line " + newPos} funcDecl.Doc = prependComment(funcDecl.Doc, comment) funcCounter++ return true diff --git a/main.go b/main.go index 33e5a36..73b5aa9 100644 --- a/main.go +++ b/main.go @@ -127,8 +127,9 @@ var ( ) const ( - garbleMapHeaderName = "garble/nameMap" - garbleSrcHeaderName = "garble/src" + // Note that these are capped at 16 bytes. + headerPrivateNameMap = "garble/privMap" + headerDebugSource = "garble/debugSrc" ) func garbledImport(path string) (*types.Package, error) { @@ -264,64 +265,13 @@ func mainErr(args []string) error { } fmt.Println(version) return nil - case "build", "test": - if !goVersionOK() { - return errJustExit - } - // Split the flags from the package arguments, since we'll need - // to run 'go list' on the same set of packages. - flags, args := splitFlagsFromArgs(args) - for _, f := range flags { - switch f { - case "-h", "-help", "--help": - return flag.ErrHelp - } - } - - err := setOptions() - if err != nil { - return err - } - - // Note that we also need to pass build flags to 'go list', such - // as -tags. - cache.BuildFlags = filterBuildFlags(flags) - if command == "test" { - cache.BuildFlags = append(cache.BuildFlags, "-test") - } - - if err := setGoPrivate(); err != nil { - return err - } - - if err := setListedPackages(args); err != nil { - return err - } - cache.ExecPath, err = os.Executable() + case "reverse": + return commandReverse(args) + case "build", "test", "list": + cmd, err := toolexecCmd(command, args) if err != nil { return err } - - if sharedTempDir, err = saveShared(); err != nil { - return err - } - os.Setenv("GARBLE_SHARED", sharedTempDir) - defer os.Remove(sharedTempDir) - - goArgs := []string{ - command, - "-trimpath", - "-toolexec=" + cache.ExecPath, - } - if command == "test" { - // vet is generally not useful on garbled code; keep it - // disabled by default. - goArgs = append(goArgs, "-vet=off") - } - goArgs = append(goArgs, flags...) - goArgs = append(goArgs, args...) - - cmd := exec.Command("go", goArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() @@ -373,6 +323,73 @@ func mainErr(args []string) error { return nil } +// toolexecCmd builds an *exec.Cmd which is set up for running "go " +// with -toolexec=garble and the supplied arguments. +// +// Note that it uses and modifies global state; in general, it should only be +// called once from mainErr in the top-level garble process. +func toolexecCmd(command string, args []string) (*exec.Cmd, error) { + if !goVersionOK() { + return nil, errJustExit + } + // Split the flags from the package arguments, since we'll need + // to run 'go list' on the same set of packages. + flags, args := splitFlagsFromArgs(args) + for _, f := range flags { + switch f { + case "-h", "-help", "--help": + return nil, flag.ErrHelp + } + } + + if err := setOptions(); err != nil { + return nil, err + } + + // Note that we also need to pass build flags to 'go list', such + // as -tags. + cache.BuildFlags = filterBuildFlags(flags) + if command == "test" { + cache.BuildFlags = append(cache.BuildFlags, "-test") + } + + if err := setGoPrivate(); err != nil { + return nil, err + } + + var err error + cache.ExecPath, err = os.Executable() + if err != nil { + return nil, err + } + + if err := setListedPackages(args); err != nil { + return nil, err + } + + sharedTempDir, err = saveShared() + if err != nil { + return nil, err + } + os.Setenv("GARBLE_SHARED", sharedTempDir) + defer os.Remove(sharedTempDir) + + goArgs := []string{ + command, + "-trimpath", + "-toolexec=" + cache.ExecPath, + } + if command == "test" { + // vet is generally not useful on garbled code; keep it + // disabled by default. + goArgs = append(goArgs, "-vet=off") + } + goArgs = append(goArgs, flags...) + goArgs = append(goArgs, args...) + + return exec.Command("go", goArgs...), nil +} + var transformFuncs = map[string]func([]string) (args []string, post func() error, _ error){ "compile": transformCompile, "link": transformLink, @@ -467,9 +484,8 @@ func transformCompile(args []string) ([]string, func() error, error) { for i, file := range files { name := filepath.Base(filepath.Clean(paths[i])) - cgoFile := strings.HasPrefix(name, "_cgo_") - comments, file := transformLineInfo(file, cgoFile) + comments, file := tf.transformLineInfo(file, name) tf.handleDirectives(comments) detachedComments[i], files[i] = comments, file @@ -567,7 +583,7 @@ func transformCompile(args []string) ([]string, func() error, error) { return err } - data, err := json.Marshal(tf.privateNameMap) + nameMap, err := json.Marshal(tf.privateNameMap) if err != nil { return err } @@ -576,12 +592,12 @@ func transformCompile(args []string) ([]string, func() error, error) { // and shouldn't break other tools like the linker since our header name is unique pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{ArchiveHeader: goobj2.ArchiveHeader{ - Name: garbleMapHeaderName, - Size: int64(len(data)), - Data: data, + Name: headerPrivateNameMap, + Size: int64(len(nameMap)), + Data: nameMap, }}, goobj2.ArchiveMember{ArchiveHeader: goobj2.ArchiveHeader{ - Name: garbleSrcHeaderName, + Name: headerDebugSource, Size: int64(obfSrcArchive.Len()), Data: obfSrcArchive.Bytes(), }}, diff --git a/reverse.go b/reverse.go new file mode 100644 index 0000000..88702ba --- /dev/null +++ b/reverse.go @@ -0,0 +1,149 @@ +// Copyright (c) 2019, The Garble Authors. +// See LICENSE for licensing information. + +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "strings" +) + +// commandReverse implements "garble reverse". +func commandReverse(args []string) error { + flags, args := splitFlagsFromArgs(args) + mainPkg := "." + if len(args) > 0 { + mainPkg = args[0] + args = args[1:] + } + + listArgs := []string{ + "-json", + "-deps", + "-export", + } + listArgs = append(listArgs, flags...) + listArgs = append(listArgs, mainPkg) + cmd, err := toolexecCmd("list", listArgs) + if err != nil { + return err + } + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + return fmt.Errorf("go list error: %v", err) + } + mainPkgPath := "" + dec := json.NewDecoder(stdout) + var privatePkgPaths []string + for dec.More() { + var pkg listedPackage + if err := dec.Decode(&pkg); err != nil { + return err + } + if pkg.Export == "" { + continue + } + if pkg.Name == "main" { + if mainPkgPath != "" { + return fmt.Errorf("found two main packages: %s %s", mainPkgPath, pkg.ImportPath) + } + mainPkgPath = pkg.ImportPath + } + if isPrivate(pkg.ImportPath) { + privatePkgPaths = append(privatePkgPaths, pkg.ImportPath) + } + buildID, err := buildidOf(pkg.Export) + if err != nil { + return err + } + // Adding it to buildInfo.imports allows us to reuse the + // "if" branch below. Plus, if this edge case triggers + // multiple times in a single package compile, we can + // call "go list" once and cache its result. + buildInfo.imports[pkg.ImportPath] = importedPkg{ + packagefile: pkg.Export, + actionID: decodeHash(splitActionID(buildID)), + } + } + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes()) + } + + var replaces []string + + for _, pkgPath := range privatePkgPaths { + ipkg := buildInfo.imports[pkgPath] + + // All original exported names names are hashed with the + // obfuscated package's action ID. + tpkg, err := origImporter.Import(pkgPath) + if err != nil { + return err + } + pkgScope := tpkg.Scope() + for _, name := range pkgScope.Names() { + obj := pkgScope.Lookup(name) + if !obj.Exported() { + continue + } + replaces = append(replaces, hashWith(ipkg.actionID, name), name) + } + } + repl := strings.NewReplacer(replaces...) + + // TODO: return a non-zero status code if we could not reverse any string. + if len(args) == 0 { + return reverseContent(os.Stdout, os.Stdin, repl) + } + for _, path := range args { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if err := reverseContent(os.Stdout, f, repl); err != nil { + return err + } + f.Close() // since we're in a loop + } + return nil +} + +func reverseContent(w io.Writer, r io.Reader, repl *strings.Replacer) error { + // Read line by line. + // Reading the entire content at once wouldn't be interactive, + // nor would it support large files well. + // Reading entire lines ensures we don't cut words in half. + // We use bufio.Reader instead of bufio.Scanner, + // to also obtain the newline characters themselves. + br := bufio.NewReader(r) + for { + // Note that ReadString can return a line as well as an error if + // we hit EOF without a newline. + // In that case, we still want to process the string. + line, readErr := br.ReadString('\n') + if _, err := repl.WriteString(w, line); err != nil { + return err + } + if readErr == io.EOF { + return nil + } + if readErr != nil { + return readErr + } + } +} diff --git a/shared.go b/shared.go index 1ad0e6d..7a30cdc 100644 --- a/shared.go +++ b/shared.go @@ -72,7 +72,7 @@ type options struct { Random bool } -// setOptions sets all options from the user supplied flags +// setOptions sets all options from the user supplied flags. func setOptions() error { wd, err := os.Getwd() if err != nil { @@ -140,6 +140,7 @@ type listedPackages map[string]*listedPackage // listedPackage contains information useful for obfuscating a package type listedPackage struct { + Name string ImportPath string Export string Deps []string diff --git a/testdata/scripts/reverse.txt b/testdata/scripts/reverse.txt new file mode 100644 index 0000000..c882ba2 --- /dev/null +++ b/testdata/scripts/reverse.txt @@ -0,0 +1,67 @@ +env GOPRIVATE=test/main + +# Unknown build flags should result in errors. +! garble reverse -badflag +stderr 'flag provided but not defined' + +garble build +exec ./main +cp stderr main.stderr +exec cat main.stderr + +# Ensure that the garbled panic output looks correct. +# This output is not reproducible between 'go test' runs, +# so we can't use a static golden file. +grep 'goroutine 1 \[running\]' main.stderr +! grep 'SomeFunc|test/main|main.go|lib.go' main.stderr + +stdin main.stderr +garble reverse +stdout -count=1 'SomeFunc' +# TODO: this is what we want when "reverse" is finished +# cmp stdout reverse.stdout + +# Ensure that the reversed output matches the non-garbled output. +go build -trimpath +exec ./main +cmp stderr reverse.stdout + +-- go.mod -- +module test/main + +go 1.15 +-- main.go -- +package main + +import "test/main/lib" + +func main() { + lib.SomeFunc() +} +-- lib/lib.go -- +package lib + +import ( + "os" + "regexp" + "runtime/debug" +) + +func SomeFunc() { + // Panic outputs include "0xNN" pointers and offsets which change + // between platforms. + // Strip them out here, to have portable static stdout files. + rxVariableSuffix := regexp.MustCompile(`0x[0-9a-f]+`) + + stack := debug.Stack() + stack = rxVariableSuffix.ReplaceAll(stack, []byte("0x??")) + os.Stderr.Write(stack) +} +-- reverse.stdout -- +goroutine 1 [running]: +runtime/debug.Stack(0x??, 0x??, 0x??) + runtime/debug/stack.go:24 +0x?? +test/main/lib.SomeFunc() + test/main/lib/lib.go:15 +0x?? +main.main() + test/main/main.go:6 +0x??