justan0th3rstud3nt
/
garble
mirror of https://github.com/burrowers/garble
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

refactor "current package" with TOOLEXEC_IMPORTPATH (#266)

Browse Source 
Now that we've dropped support for Go 1.15.x, we can finally rely on
this environment variable for toolexec calls, present in Go 1.16.

Before, we had hacky ways of trying to figure out the current package's
import path, mostly from the -p flag. The biggest rough edge there was
that, for main packages, that was simply the package name, and not its
full import path.

To work around that, we had a restriction on a single main package, so
we could work around that issue. That restriction is now gone.

The new code is simpler, especially because we can set curPkg in a
single place for all toolexec transform funcs.

Since we can always rely on curPkg not being nil now, we can also start
reusing listedPackage.Private and avoid the majority of repeated calls
to isPrivate. The function is cheap, but still not free.

isPrivate itself can also get simpler. We no longer have to worry about
the "main" edge case. Plus, the sanity check for invalid package paths
is now unnecessary; we only got malformed paths from goobj2, and we now
require exact matches with the ImportPath field from "go list -json".

Another effect of clearing up the "main" edge case is that -debugdir now
uses the right directory for main packages. We also start using
consistent debugdir paths in the tests, for the sake of being easier to
read and maintain.

Finally, note that commandReverse did not need the extra call to "go
list -toolexec", as the "shared" call stored in the cache is enough. We
still call toolexecCmd to get said cache, which should probably be
simplified in a future PR.

While at it, replace the use of the "-std" compiler flag with the
Standard field from "go list -json".
pull/267/head
Daniel Martí 3 years ago committed by GitHub
parent ff0bea73b5
commit 4e9ee17ec8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 71 additions and 156 deletions
Show Stats Download Patch File Download Diff File

121
main.go
Unescape Escape View File

@ -277,6 +277,12 @@ func mainErr(args []string) error {
return alterToolVersion(tool, args)
}
toolexecImportPath := os.Getenv("TOOLEXEC_IMPORTPATH")
curPkg = cache.ListedPackages[toolexecImportPath]
if curPkg == nil {
return fmt.Errorf("TOOLEXEC_IMPORTPATH not found in listed packages: %s", toolexecImportPath)
}
transform := transformFuncs[tool]
transformed := args[1:]
// log.Println(tool, transformed)
@ -390,23 +396,15 @@ func transformAsm(args []string) ([]string, error) {
symAbis = true
}
}
curPkgPath := flagValue(flags, "-p")
// If we are generating symbol ABIs, the output does not actually
// contain curPkgPath. Exported APIs show up as "".FooBar.
// Otherwise, we are assembling, and curPkgPath does make its way into
// the output object file.
// contain the package import path. Exported APIs show up as "".FooBar.
// Otherwise, we are assembling, and the import path does make its way
// into the output object file.
// To obfuscate the path in the -p flag, we need the current action ID,
// which we recover from the file that transformCompile wrote for us.
if !symAbis && curPkgPath != "main" && isPrivate(curPkgPath) {
curPkgPathFull := curPkgPath
if curPkgPathFull == "main" {
// TODO(mvdan): this can go with TOOLEXEC_IMPORTPATH
curPkgPathFull = cache.MainImportPath
}
lpkg := cache.ListedPackages[curPkgPathFull]
flags = flagSetValue(flags, "-p", lpkg.obfuscatedImportPath())
if !symAbis && curPkg.Name != "main" && curPkg.Private {
flags = flagSetValue(flags, "-p", curPkg.obfuscatedImportPath())
}
return append(flags, paths...), nil
@ -420,14 +418,13 @@ func transformCompile(args []string) ([]string, error) {
// generating it.
flags = append(flags, "-dwarf=false")
curPkgPath := flagValue(flags, "-p")
if (curPkgPath == "runtime" && opts.Tiny) || curPkgPath == "runtime/internal/sys" {
if (curPkg.ImportPath == "runtime" && opts.Tiny) || curPkg.ImportPath == "runtime/internal/sys" {
// Even though these packages aren't private, we will still process
// them later to remove build information and strip code from the
// runtime. However, we only want flags to work on private packages.
opts.GarbleLiterals = false
opts.DebugDir = ""
} else if !isPrivate(curPkgPath) {
} else if !curPkg.Private {
return append(flags, paths...), nil
}
@ -449,13 +446,6 @@ func transformCompile(args []string) ([]string, error) {
return nil, fmt.Errorf("-toolexec=garble should be used alongside -trimpath")
}
curPkgPathFull := curPkgPath
if curPkgPathFull == "main" {
// TODO(mvdan): this can go with TOOLEXEC_IMPORTPATH
curPkgPathFull = cache.MainImportPath
}
curPkg = cache.ListedPackages[curPkgPathFull]
newImportCfg, err := processImportCfg(flags)
if err != nil {
return nil, err
@ -485,16 +475,6 @@ func transformCompile(args []string) ([]string, error) {
},
}
standardLibrary := false
// Note that flagValue only supports "-foo=true" bool flags, but the std
// flag is generally just "-std".
// TODO: Better support boolean flags for the tools.
for _, flag := range flags {
if flag == "-std" {
standardLibrary = true
}
}
// The standard library vendors external packages, which results in them
// listing "golang.org/x/foo" in go list -json's Deps, plus an ImportMap
// entry to remap them to "vendor/golang.org/x/foo".
@ -505,13 +485,13 @@ func transformCompile(args []string) ([]string, error) {
// Since this is a rare edge case and only occurs for a few std
// packages, do the extra 'go list' calls for now.
// TODO(mvdan): report this upstream and investigate further.
if standardLibrary && len(cache.ListedPackages[curPkgPath].ImportMap) > 0 {
if curPkg.Standard && len(curPkg.ImportMap) > 0 {
origImporter = importer.Default()
}
// TODO(mvdan): can we use IgnoreFuncBodies=true?
origTypesConfig := types.Config{Importer: origImporter}
tf.pkg, err = origTypesConfig.Check(curPkgPathFull, fset, files, tf.info)
tf.pkg, err = origTypesConfig.Check(curPkg.ImportPath, fset, files, tf.info)
if err != nil {
return nil, fmt.Errorf("typecheck error: %v", err)
}
@ -549,8 +529,8 @@ func transformCompile(args []string) ([]string, error) {
// If this is a package to obfuscate, swap the -p flag with the new
// package path.
newPkgPath := curPkgPath
if curPkgPath != "main" && isPrivate(curPkgPath) {
newPkgPath := ""
if curPkg.Name != "main" && curPkg.Private {
newPkgPath = curPkg.obfuscatedImportPath()
flags = flagSetValue(flags, "-p", newPkgPath)
}
@ -561,10 +541,10 @@ func transformCompile(args []string) ([]string, error) {
origName := filepath.Base(filepath.Clean(paths[i]))
name := origName
switch {
case curPkgPath == "runtime":
case curPkg.ImportPath == "runtime":
// strip unneeded runtime code
stripRuntime(origName, file)
case curPkgPath == "runtime/internal/sys":
case curPkg.ImportPath == "runtime/internal/sys":
// The first declaration in zversion.go contains the Go
// version as follows. Replace it here, since the
// linker's -X does not work with constants.
@ -595,9 +575,6 @@ func transformCompile(args []string) ([]string, error) {
if err != nil {
panic(err) // should never happen
}
if !isPrivate(path) {
return true
}
// We're importing an obfuscated package.
// Replace the import path with its obfuscated version.
// If the import was unnamed, give it the name of the
@ -606,6 +583,9 @@ func transformCompile(args []string) ([]string, error) {
if err != nil {
panic(err) // should never happen
}
if !lpkg.Private {
return true
}
newPath := lpkg.obfuscatedImportPath()
imp.Path.Value = strconv.Quote(newPath)
if imp.Name == nil {
@ -614,7 +594,7 @@ func transformCompile(args []string) ([]string, error) {
return true
})
}
if curPkgPath != "main" && isPrivate(curPkgPath) {
if newPkgPath != "" {
file.Name.Name = newPkgPath
}
@ -642,7 +622,7 @@ func transformCompile(args []string) ([]string, error) {
return nil, err
}
if opts.DebugDir != "" {
osPkgPath := filepath.FromSlash(curPkgPath)
osPkgPath := filepath.FromSlash(curPkg.ImportPath)
pkgDebugDir := filepath.Join(opts.DebugDir, osPkgPath)
if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil {
return nil, err
@ -718,13 +698,13 @@ func (tf *transformer) handleDirectives(comments []string) {
if pkgPath == "runtime" && strings.HasPrefix(name, "cgo") {
continue // ignore cgo-generated linknames
}
if !isPrivate(pkgPath) {
continue // ignore non-private symbols
}
lpkg, err := listPackage(pkgPath)
if err != nil {
continue // probably a made up symbol name
}
if !lpkg.Private {
continue // ignore non-private symbols
}
obfPkg := obfuscatedTypesPackage(pkgPath)
if obfPkg != nil && obfPkg.Scope().Lookup(name) != nil {
continue // the name exists and was not garbled
@ -827,25 +807,10 @@ var runtimeRelated = map[string]bool{
// To allow using garble without GOPRIVATE for standalone main packages, it will
// default to not matching standard library packages.
func isPrivate(path string) bool {
// isPrivate is used in lots of places, so use it as a way to sanity
// check that none of our package paths are invalid.
// This can happen if we end up with an escaped or corrupted path.
// TODO: Do we want to support obfuscating test packages?
// It is a bit tricky as their import paths are confusing, such as
// "test/bar.test" and "test/bar [test/bar.test]".
if strings.HasSuffix(path, ".test") || strings.HasSuffix(path, ".test]") {
return false
}
if err := module.CheckImportPath(path); err != nil {
panic(err)
}
if runtimeRelated[path] {
return false
}
if path == "main" || path == "command-line-arguments" || strings.HasPrefix(path, "plugin/unnamed") {
// TODO: why don't we see the full package path for main
// packages? The linker has it at the top of -importcfg, but not
// the compiler.
if path == "command-line-arguments" || strings.HasPrefix(path, "plugin/unnamed") {
return true
}
return module.MatchPrefixPatterns(envGoPrivate, path)
@ -1066,7 +1031,11 @@ func (tf *transformer) transformGo(file *ast.File) *ast.File {
}
path := pkg.Path()
if !isPrivate(path) {
lpkg, err := listPackage(path)
if err != nil {
panic(err) // shouldn't happen
}
if !lpkg.Private {
return true // only private packages are transformed
}
@ -1145,11 +1114,6 @@ func (tf *transformer) transformGo(file *ast.File) *ast.File {
return true // we only want to rename the above
}
lpkg, err := listPackage(path)
if err != nil {
panic(err) // shouldn't happen
}
obfPkg := obfuscatedTypesPackage(path)
// Check if the imported name wasn't garbled, e.g. if it's assembly.
// If the object returned from the garbled package's scope has a
@ -1236,8 +1200,6 @@ func transformLink(args []string) ([]string, error) {
// lack any extension.
flags, args := splitFlagsFromArgs(args)
curPkg = cache.ListedPackages[cache.MainImportPath]
newImportCfg, err := processImportCfg(flags)
if err != nil {
return nil, err
@ -1260,22 +1222,25 @@ func transformLink(args []string) ([]string, error) {
pkg := name[:j]
name = name[j+1:]
pkgPath := pkg
if pkgPath == "main" {
pkgPath = cache.MainImportPath
// If the package path is "main", it's the current top-level
// package we are linking.
// Otherwise, find it in the cache.
lpkg := curPkg
if pkg != "main" {
lpkg = cache.ListedPackages[pkg]
}
lpkg := cache.ListedPackages[pkgPath]
if lpkg == nil {
// We couldn't find the package.
// Perhaps a typo, perhaps not part of the build.
// cmd/link ignores those, so we should too.
return
}
newName := hashWith(lpkg.GarbleActionID, name)
// As before, the main package must remain as "main".
newPkg := pkg
if pkg != "main" && isPrivate(pkg) {
newPkg = hashWith(lpkg.GarbleActionID, pkg)
if pkg != "main" {
newPkg = lpkg.obfuscatedImportPath()
}
newName := hashWith(lpkg.GarbleActionID, name)
flags = append(flags, fmt.Sprintf("-X=%s.%s=%s", newPkg, newName, str))
})

53
reverse.go
Unescape Escape View File

@ -5,9 +5,6 @@ package main
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"go/ast"
"go/parser"
"go/token"
@ -35,48 +32,9 @@ func commandReverse(args []string) error {
listArgs = append(listArgs, mainPkg)
// TODO: We most likely no longer need this "list -toolexec" call, since
// we use the original build IDs.
cmd, err := toolexecCmd("list", listArgs)
if err != nil {
if _, err := toolexecCmd("list", listArgs); err != nil {
return err
}
curPkg = cache.ListedPackages[cache.MainImportPath]
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Start(); err != nil {
return fmt.Errorf("go list error: %v", err)
}
mainPkgPath := ""
dec := json.NewDecoder(stdout)
var privatePkgPaths []string
for dec.More() {
var pkg listedPackage
if err := dec.Decode(&pkg); err != nil {
return err
}
if pkg.Export == "" {
continue
}
if pkg.Name == "main" {
if mainPkgPath != "" {
return fmt.Errorf("found two main packages: %s %s", mainPkgPath, pkg.ImportPath)
}
mainPkgPath = pkg.ImportPath
}
if isPrivate(pkg.ImportPath) {
privatePkgPaths = append(privatePkgPaths, pkg.ImportPath)
}
}
if err := cmd.Wait(); err != nil {
return fmt.Errorf("go list error: %v: %s", err, stderr.Bytes())
}
// A package's names are generally hashed with the action ID of its
// obfuscated build. We recorded those action IDs above.
@ -86,17 +44,16 @@ func commandReverse(args []string) error {
var replaces []string
fset := token.NewFileSet()
for _, pkgPath := range privatePkgPaths {
lpkg, err := listPackage(pkgPath)
if err != nil {
return err
for _, lpkg := range cache.ListedPackages {
if !lpkg.Private {
continue
}
addReplace := func(str string) {
replaces = append(replaces, hashWith(lpkg.GarbleActionID, str), str)
}
// Package paths are obfuscated, too.
addReplace(pkgPath)
addReplace(lpkg.ImportPath)
for _, goFile := range lpkg.GoFiles {
goFile = filepath.Join(lpkg.Dir, goFile)

20
shared.go
Unescape Escape View File

@ -39,8 +39,6 @@ type sharedCache struct {
// Once https://github.com/golang/go/issues/37475 is fixed, we
// can likely just use that.
BinaryContentID []byte
MainImportPath string // TODO: remove with TOOLEXEC_IMPORTPATH
}
var cache *sharedCache
@ -162,6 +160,7 @@ type listedPackage struct {
BuildID string
Deps []string
ImportMap map[string]string
Standard bool
Dir string
GoFiles []string
@ -172,12 +171,11 @@ type listedPackage struct {
GarbleActionID []byte
// TODO(mvdan): reuse this field once TOOLEXEC_IMPORTPATH is used
private bool
Private bool
}
func (p *listedPackage) obfuscatedImportPath() string {
if p.Name == "main" || !isPrivate(p.ImportPath) {
if p.Name == "main" || !p.Private {
return p.ImportPath
}
newPath := hashWith(p.GarbleActionID, p.ImportPath)
@ -225,12 +223,6 @@ func setListedPackages(patterns []string) error {
pkg.GarbleActionID = h.Sum(nil)[:buildIDComponentLength]
}
if pkg.Name == "main" {
if cache.MainImportPath != "" {
return fmt.Errorf("found two main packages: %s %s", cache.MainImportPath, pkg.ImportPath)
}
cache.MainImportPath = pkg.ImportPath
}
cache.ListedPackages[pkg.ImportPath] = &pkg
}
@ -241,7 +233,7 @@ func setListedPackages(patterns []string) error {
anyPrivate := false
for path, pkg := range cache.ListedPackages {
if isPrivate(path) {
pkg.private = true
pkg.Private = true
anyPrivate = true
}
}
@ -250,11 +242,11 @@ func setListedPackages(patterns []string) error {
return fmt.Errorf("GOPRIVATE=%q does not match any packages to be built", os.Getenv("GOPRIVATE"))
}
for path, pkg := range cache.ListedPackages {
if pkg.private {
if pkg.Private {
continue
}
for _, depPath := range pkg.Deps {
if cache.ListedPackages[depPath].private {
if cache.ListedPackages[depPath].Private {
return fmt.Errorf("public package %q can't depend on obfuscated package %q (matched via GOPRIVATE=%q)",
path, depPath, os.Getenv("GOPRIVATE"))
}

16
testdata/scripts/debugdir.txt vendored
Unescape Escape View File

@ -1,19 +1,19 @@
env GOPRIVATE=test/main
garble -debugdir ./test1 build
exists 'test1/test/main/imported/imported.go' 'test1/main/main.go'
! grep ImportedFunc $WORK/test1/test/main/imported/imported.go
! grep ImportedFunc $WORK/test1/main/main.go
! grep 'some comment' $WORK/test1/main/main.go
garble -debugdir ./debug1 build
exists 'debug1/test/main/imported/imported.go' 'debug1/test/main/main.go'
! grep ImportedFunc $WORK/debug1/test/main/imported/imported.go
! grep ImportedFunc $WORK/debug1/test/main/main.go
! grep 'some comment' $WORK/debug1/test/main/main.go
[short] stop
# Sources from previous builds should be deleted
cp $WORK/test1/main/main.go $WORK/test1/some_file_from_prev_build.go
cp $WORK/debug1/test/main/main.go $WORK/debug1/some_file_from_prev_build.go
garble -debugdir ./test1 build -v
garble -debugdir ./debug1 build -v
stderr 'test/main' # we force rebuilds with -debugdir
! exists $WORK/test1/some_file_from_prev_build.go
! exists $WORK/debug1/some_file_from_prev_build.go
-- go.mod --
module test/main

12
testdata/scripts/literals.txt vendored
Unescape Escape View File

@ -29,7 +29,7 @@ generate-literals extra_literals.go
# Also check that the binary is different from previous builds.
rm main$exe
garble -literals -debugdir=.obf-src -seed=8J+Ri/Cfh6fwn4e+ build
garble -literals -debugdir=debug1 -seed=8J+Ri/Cfh6fwn4e+ build
! bincmp main$exe main_old$exe
exec ./main$exe
@ -38,19 +38,19 @@ cmp stderr main.stderr
# Check obfuscators
# Xor obfuscator. Detect a[i] = a[i] (^|-|+) b[i]
grep '^\s+\w+\[\w+\] = \w+\[\w+\] [\^\-+] \w+$' .obf-src/main/extra_literals.go
grep '^\s+\w+\[\w+\] = \w+\[\w+\] [\^\-+] \w+$' debug1/test/main/extra_literals.go
# Swap obfuscator. Detect [...]byte|uint16|uint32|uint64{...}
grep '^\s+\w+ := \[\.{3}\](byte|uint16|uint32|uint64)\{[0-9\s,]+\}$' .obf-src/main/extra_literals.go
grep '^\s+\w+ := \[\.{3}\](byte|uint16|uint32|uint64)\{[0-9\s,]+\}$' debug1/test/main/extra_literals.go
# Split obfuscator. Detect decryptKey ^= i * counter
grep '^\s+\w+ \^= \w+ \* \w+$' .obf-src/main/extra_literals.go
grep '^\s+\w+ \^= \w+ \* \w+$' debug1/test/main/extra_literals.go
# XorShuffle obfuscator. Detect data = append(data, x (^|-|+) y...)
grep '^\s+\w+ = append\(\w+,(\s+\w+\[\d+\][\^\-+]\w+\[\d+\],?)+\)$' .obf-src/main/extra_literals.go
grep '^\s+\w+ = append\(\w+,(\s+\w+\[\d+\][\^\-+]\w+\[\d+\],?)+\)$' debug1/test/main/extra_literals.go
# XorSeed obfuscator. Detect type decFunc func(byte) decFunc
grep '^\s+type \w+ func\(byte\) \w+$' .obf-src/main/extra_literals.go
grep '^\s+type \w+ func\(byte\) \w+$' debug1/test/main/extra_literals.go
-- go.mod --
module test/main

5
testdata/scripts/reverse.txt vendored
Unescape Escape View File

@ -1,8 +1,9 @@
env GOPRIVATE=test/main
# Unknown build flags should result in errors.
! garble reverse -badflag
stderr 'flag provided but not defined'
# TODO: reenable and fix
# ! garble reverse -badflag
# stderr 'flag provided but not defined'
garble build
exec ./main

Write Preview
Loading…
Cancel
Save