From 2735555ab262342b26659e1c1c6f0091335478b6 Mon Sep 17 00:00:00 2001 From: pagran <67878280+pagran@users.noreply.github.com> Date: Fri, 14 Aug 2020 21:47:15 +0300 Subject: [PATCH] Update filename and add line number obfuscation (#94) Fixes #2. Line numbers are now obfuscated, via `//line` comments. Filenames are now obfuscated via `//line` comments, instead of changing the actual filename. New flag `-tiny` to reduce the binary size, at the cost of reversibility. --- line_obfuscator.go | 109 ++++++++++++++++++++++++++++++++++ main.go | 70 ++++++++++++---------- testdata/scripts/basic.txt | 4 +- testdata/scripts/debugdir.txt | 7 +-- testdata/scripts/literals.txt | 10 ++-- testdata/scripts/syntax.txt | 2 +- testdata/scripts/tiny.txt | 41 +++++++++++++ 7 files changed, 198 insertions(+), 45 deletions(-) create mode 100644 line_obfuscator.go create mode 100644 testdata/scripts/tiny.txt diff --git a/line_obfuscator.go b/line_obfuscator.go new file mode 100644 index 0000000..cf43cee --- /dev/null +++ b/line_obfuscator.go @@ -0,0 +1,109 @@ +package main + +import ( + "fmt" + "go/ast" + mathrand "math/rand" + "strings" + + "golang.org/x/tools/go/ast/astutil" +) + +const ( + // PosMax is the largest line or column value that can be represented without loss. + // Source: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/internal/syntax/pos.go#11 + PosMax = 1 << 30 + + // PosMin is the smallest correct value for the line number. + // Source: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/internal/syntax/parser_test.go#229 + PosMin = 1 +) + +func prependComment(group *ast.CommentGroup, comment *ast.Comment) *ast.CommentGroup { + if group == nil { + return &ast.CommentGroup{List: []*ast.Comment{comment}} + } + + group.List = append([]*ast.Comment{comment}, group.List...) + return group +} + +// Remove all comments from CommentGroup except //go: directives. +func clearCommentGroup(group *ast.CommentGroup) *ast.CommentGroup { + if group == nil { + return nil + } + + var comments []*ast.Comment + + for _, comment := range group.List { + if strings.HasPrefix(comment.Text, "//go:") { + comments = append(comments, &ast.Comment{Text: comment.Text}) + } + } + if len(comments) == 0 { + return nil + } + return &ast.CommentGroup{List: comments} +} + +// Remove all comments from Doc (if any) except //go: directives. +func clearNodeComments(node ast.Node) { + switch n := node.(type) { + case *ast.Field: + n.Doc = clearCommentGroup(n.Doc) + case *ast.ImportSpec: + n.Doc = clearCommentGroup(n.Doc) + case *ast.ValueSpec: + n.Doc = clearCommentGroup(n.Doc) + case *ast.TypeSpec: + n.Doc = clearCommentGroup(n.Doc) + case *ast.GenDecl: + n.Doc = clearCommentGroup(n.Doc) + case *ast.FuncDecl: + n.Doc = clearCommentGroup(n.Doc) + case *ast.File: + n.Doc = clearCommentGroup(n.Doc) + } +} + +func findBuildTags(commentGroups []*ast.CommentGroup) (buildTags []string) { + for _, group := range commentGroups { + for _, comment := range group.List { + if !strings.Contains(comment.Text, "+build") { + continue + } + buildTags = append(buildTags, comment.Text) + } + } + return buildTags +} + +func transformLineInfo(fileIndex int, file *ast.File) ([]string, *ast.File) { + // Save build tags and add file name leak protection + extraComments := append(findBuildTags(file.Comments), "", "//line :1") + + file.Comments = nil + pre := func(cursor *astutil.Cursor) bool { + node := cursor.Node() + clearNodeComments(node) + + funcDecl, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + + if envGarbleTiny { + funcDecl.Doc = prependComment(funcDecl.Doc, &ast.Comment{Text: "//line :1"}) + return true + } + + // TODO: Optimize the generated values of line numbers to reduce space usage. + linePos := hashWithAsUint64(buildInfo.buildID, fmt.Sprintf("%d:%s", fileIndex, funcDecl.Name), PosMin, PosMax) + comment := &ast.Comment{Text: fmt.Sprintf("//line %c.go:%d", nameCharset[mathrand.Intn(len(nameCharset))], linePos)} + funcDecl.Doc = prependComment(funcDecl.Doc, comment) + return true + } + + return extraComments, astutil.Apply(file, pre, nil).(*ast.File) +} diff --git a/main.go b/main.go index 3483a36..cce416f 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ var flagSet = flag.NewFlagSet("garble", flag.ContinueOnError) var ( flagGarbleLiterals bool + flagGarbleTiny bool flagDebugDir string flagSeed string ) @@ -44,6 +45,7 @@ var ( func init() { flagSet.Usage = usage flagSet.BoolVar(&flagGarbleLiterals, "literals", false, "Encrypt all literals with AES, currently only literal strings are supported") + flagSet.BoolVar(&flagGarbleTiny, "tiny", false, "Optimize for binary size, losing the ability to reverse the process") flagSet.StringVar(&flagDebugDir, "debugdir", "", "Write the garbled source to a given directory: '-debugdir=./debug'") flagSet.StringVar(&flagSeed, "seed", "", "Provide a custom base64-encoded seed: '-seed=o9WDTZ4CN4w=' \nFor a random seed provide: '-seed=random'") } @@ -72,7 +74,8 @@ var ( deferred []func() error fset = token.NewFileSet() - b64 = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_z") + nameCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_z" + b64 = base64.NewEncoding(nameCharset) printConfig = printer.Config{Mode: printer.RawFormat} // listPackage helps implement a types.Importer which finds the export @@ -94,6 +97,7 @@ var ( envGarbleDir = os.Getenv("GARBLE_DIR") envGarbleLiterals = os.Getenv("GARBLE_LITERALS") == "true" + envGarbleTiny = os.Getenv("GARBLE_TINY") == "true" envGarbleDebugDir = os.Getenv("GARBLE_DEBUGDIR") envGarbleSeed = os.Getenv("GARBLE_SEED") envGoPrivate string // filled via 'go env' below to support 'go env -w' @@ -247,6 +251,7 @@ func mainErr(args []string) error { } os.Setenv("GARBLE_DIR", wd) os.Setenv("GARBLE_LITERALS", fmt.Sprint(flagGarbleLiterals)) + os.Setenv("GARBLE_TINY", fmt.Sprint(flagGarbleTiny)) if flagSeed == "random" { seed = make([]byte, 16) // random 128 bit seed @@ -478,6 +483,7 @@ func transformCompile(args []string) ([]string, error) { // TODO: randomize the order and names of the files newPaths := make([]string, 0, len(files)) for i, file := range files { + var extraComments []string origName := filepath.Base(filepath.Clean(paths[i])) name := origName switch { @@ -504,8 +510,8 @@ func transformCompile(args []string) ([]string, error) { // messy. name = "_cgo_" + name default: + extraComments, file = transformLineInfo(i, file) file = transformGo(file, info, blacklist) - name = fmt.Sprintf("z%d.go", i) // Uncomment for some quick debugging. Do not delete. // fmt.Fprintf(os.Stderr, "\n-- %s/%s --\n", pkgPath, origName) @@ -531,6 +537,13 @@ func transformCompile(args []string) ([]string, error) { printWriter = io.MultiWriter(tempFile, debugFile) } + if len(extraComments) > 0 { + for _, comment := range extraComments { + if _, err = printWriter.Write([]byte(comment + "\n")); err != nil { + return nil, err + } + } + } if err := printConfig.Fprint(printWriter, fset, file); err != nil { return nil, err } @@ -654,6 +667,16 @@ func hashWith(salt, value string) string { return "z" + sum[:length] } +func hashWithAsUint64(salt, value string, min, max uint64) uint64 { + d := sha256.New() + io.WriteString(d, salt) + d.Write(seed) + io.WriteString(d, value) + sum := d.Sum(nil) + val := binary.LittleEndian.Uint64(sum) + return min + (val % (max - min)) +} + // buildBlacklist collects all the objects in a package which are known to be // used with reflect.TypeOf or reflect.ValueOf. Since we obfuscate one package // at a time, we only detect those if the type definition and the reflect usage @@ -720,38 +743,19 @@ func buildBlacklist(files []*ast.File, info *types.Info, pkg *types.Package) map // transformGo garbles the provided Go syntax node. func transformGo(file *ast.File, info *types.Info, blacklist map[types.Object]struct{}) *ast.File { - // Remove all comments, minus the "//go:" compiler directives. - // The final binary should still not contain comment text, but removing - // it helps ensure that (and makes position info less predictable). - origComments := file.Comments - file.Comments = nil - for _, commentGroup := range origComments { - for _, comment := range commentGroup.List { - if strings.HasPrefix(comment.Text, "//go:") { - file.Comments = append(file.Comments, &ast.CommentGroup{ - List: []*ast.Comment{comment}, - }) - } + // Shuffle top level declarations + mathrand.Shuffle(len(file.Decls), func(i, j int) { + decl1 := file.Decls[i] + decl2 := file.Decls[j] + + // Import declarations must remain at the top of the file. + gd1, ok1 := decl1.(*ast.GenDecl) + gd2, ok2 := decl2.(*ast.GenDecl) + if (ok1 && gd1.Tok == token.IMPORT) || (ok2 && gd2.Tok == token.IMPORT) { + return } - } - - // Shuffle top level declarations if there are no remaining compiler - // directives. - if len(file.Comments) == 0 { - // TODO: Also allow files with compiler directives. - mathrand.Shuffle(len(file.Decls), func(i, j int) { - decl1 := file.Decls[i] - decl2 := file.Decls[j] - - // Import declarations must remain at the top of the file. - gd1, ok1 := decl1.(*ast.GenDecl) - gd2, ok2 := decl2.(*ast.GenDecl) - if (ok1 && gd1.Tok == token.IMPORT) || (ok2 && gd2.Tok == token.IMPORT) { - return - } - file.Decls[i], file.Decls[j] = decl2, decl1 - }) - } + file.Decls[i], file.Decls[j] = decl2, decl1 + }) pre := func(cursor *astutil.Cursor) bool { node, ok := cursor.Node().(*ast.Ident) diff --git a/testdata/scripts/basic.txt b/testdata/scripts/basic.txt index 3f42075..932cb94 100644 --- a/testdata/scripts/basic.txt +++ b/testdata/scripts/basic.txt @@ -27,7 +27,7 @@ stdout 'unknown' ! stdout $gofullversion # The binary can't contain the version string either. -! binsubstr main$exe ${WORK@R} 'globalVar' 'globalFunc' $gofullversion +! binsubstr main$exe ${WORK@R} 'main.go' 'globalVar' 'globalFunc' $gofullversion [short] stop # checking that the build is reproducible is slow @@ -50,7 +50,7 @@ cmp stderr main.stderr # The default build includes full non-trimmed paths, as well as our names. # Only check $WORK on non-windows, because it's difficult to do it there. -binsubstr main$exe 'globalVar' 'globalFunc' $gofullversion +binsubstr main$exe 'main.go' 'globalVar' 'globalFunc' $gofullversion [!windows] binsubstr main$exe ${WORK@R} -- main.go -- diff --git a/testdata/scripts/debugdir.txt b/testdata/scripts/debugdir.txt index ab15769..08df254 100644 --- a/testdata/scripts/debugdir.txt +++ b/testdata/scripts/debugdir.txt @@ -1,8 +1,7 @@ garble -debugdir ./test1 build -exists 'test1/test/imported/z0.go' 'test1/main/z0.go' -! grep ImportedFunc $WORK/test1/test/imported/z0.go -! grep ImportedFunc $WORK/test1/main/z0.go - +exists 'test1/test/imported/imported.go' 'test1/main/main.go' +! grep ImportedFunc $WORK/test1/test/imported/imported.go +! grep ImportedFunc $WORK/test1/main/main.go -- go.mod -- module test diff --git a/testdata/scripts/literals.txt b/testdata/scripts/literals.txt index 7ffb884..13109d5 100644 --- a/testdata/scripts/literals.txt +++ b/testdata/scripts/literals.txt @@ -37,19 +37,19 @@ cmp stderr normal.stderr # Check obfuscators # Xor obfuscator. Detect a[i] = a[i] (^|-|+) b[i] -grep '^\s+\w+\[\w+\] = \w+\[\w+\] [\^\-+] \w+$' .obf-src/main/z0.go +grep '^\s+\w+\[\w+\] = \w+\[\w+\] [\^\-+] \w+$' .obf-src/main/extraLiterals.go # Swap obfuscator. Detect [...]byte|uint16|uint32|uint64{...} -grep '^\s+\w+ := \[\.{3}\](byte|uint16|uint32|uint64)\{[0-9\s,]+\}$' .obf-src/main/z0.go +grep '^\s+\w+ := \[\.{3}\](byte|uint16|uint32|uint64)\{[0-9\s,]+\}$' .obf-src/main/extraLiterals.go # Split obfuscator. Detect decryptKey ^= i * counter -grep '^\s+\w+ \^= \w+ \* \w+$' .obf-src/main/z0.go +grep '^\s+\w+ \^= \w+ \* \w+$' .obf-src/main/extraLiterals.go # XorShuffle obfuscator. Detect data = append(data, x (^|-|+) y...) -grep '^\s+\w+ = append\(\w+,(\s+\w+\[\d+\][\^\-+]\w+\[\d+\],?)+\)$' .obf-src/main/z0.go +grep '^\s+\w+ = append\(\w+,(\s+\w+\[\d+\][\^\-+]\w+\[\d+\],?)+\)$' .obf-src/main/extraLiterals.go # XorSeed obfuscator. Detect type decFunc func(byte) decFunc -grep '^\s+type \w+ func\(byte\) \w+$' .obf-src/main/z0.go +grep '^\s+type \w+ func\(byte\) \w+$' .obf-src/main/extraLiterals.go -- go.mod -- diff --git a/testdata/scripts/syntax.txt b/testdata/scripts/syntax.txt index 7173a59..7e7da57 100644 --- a/testdata/scripts/syntax.txt +++ b/testdata/scripts/syntax.txt @@ -11,7 +11,7 @@ cmp stderr main.stderr ! binsubstr main$exe 'localName' 'globalConst' 'globalVar' 'globalType' 'valuable information' -binsubstr debug/main/z1.go 'localName' 'globalConst' +binsubstr debug/main/scopes.go 'localName' 'globalConst' -- go.mod -- module test/main diff --git a/testdata/scripts/tiny.txt b/testdata/scripts/tiny.txt new file mode 100644 index 0000000..001d2b7 --- /dev/null +++ b/testdata/scripts/tiny.txt @@ -0,0 +1,41 @@ +env TINY_PATTERN='^\/\/line :1$' +env DEFAULT_PATTERN='^\/\/line \w\.go:[1-9][0-9]*$' +env DEFAULT_STACK_PATTERN='^\t\w\.go:[1-9][0-9]*(\s\+0x[0-9a-f]+)?' +env TINY_STACK_PATTERN='^\t\?\?:[0-9][0-9]*(\s\+0x[0-9a-f]+)?$' + +# Tiny mode +garble -tiny -debugdir=.obf-src build + +grep $TINY_PATTERN .obf-src/main/main.go +! grep $DEFAULT_PATTERN .obf-src/main/main.go + +! exec ./main$exe +! stderr 'main\.go' +! stderr $DEFAULT_STACK_PATTERN +stderr $TINY_STACK_PATTERN + +[short] stop # no need to verify this with -short + +# Default mode + +garble -debugdir=.obf-src build + +# Check for file name leak protection +grep $TINY_PATTERN .obf-src/main/main.go + +# Check for default line obfuscation +grep $DEFAULT_PATTERN .obf-src/main/main.go + +! exec ./main$exe +! stderr 'main\.go' +! stderr $TINY_STACK_PATTERN +stderr $DEFAULT_STACK_PATTERN + +-- go.mod -- +module main +-- main.go -- +package main + +func main() { + panic("Test") +} \ No newline at end of file