From 803c1d94393688902bcbc8d233c158bfcba4a60c Mon Sep 17 00:00:00 2001 From: pagran Date: Thu, 22 Oct 2020 22:21:33 +0300 Subject: [PATCH] Store obfuscated sources in object files (#158) Now the flag "-debugdir" does not trigger a full recompilation. Obfuscated source files are saved to object files and are extracted during linking. --- go.mod | 2 +- go.sum | 2 + import_obfuscation.go | 77 +++++++++++++++++++++- main.go | 117 +++++++++++++++++----------------- testdata/scripts/debugdir.txt | 9 +++ 5 files changed, 148 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index 220fa9b..7b596c1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module mvdan.cc/garble go 1.15 require ( - github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7 + github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e github.com/google/go-cmp v0.5.2 github.com/rogpeppe/go-internal v1.6.3-0.20201011174404-9f985d550aa7 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 diff --git a/go.sum b/go.sum index 84eb7fe..7be540b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7 h1:W9kchICcGYOJQBZNn2P6j739QbY8OKskB3SyFZ/IRIU= github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7/go.mod h1:QzgxDLY/qdKlvnbnb65eqTedhvQPbaSP2NqIbcuKvsQ= +github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e h1:IQMhZQS5d9OUmcXgv330n31ZnD97HAZE36K7JgIjvog= +github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e/go.mod h1:QzgxDLY/qdKlvnbnb65eqTedhvQPbaSP2NqIbcuKvsQ= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= diff --git a/import_obfuscation.go b/import_obfuscation.go index f95c9ed..822c1d1 100644 --- a/import_obfuscation.go +++ b/import_obfuscation.go @@ -4,12 +4,16 @@ package main import ( + "archive/tar" "bufio" "bytes" + "compress/gzip" "encoding/json" "fmt" + "io" "io/ioutil" "os" + "path/filepath" "sort" "strings" @@ -62,6 +66,68 @@ func appendPrivateNameMap(pkg *goobj2.Package, nameMap map[string]string) error return nil } +// extractDebugObfSrc extracts obfuscated sources from object files if -debugdir flag is enabled. +func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error { + if envGarbleDebugDir == "" { + return nil + } + + var archiveMember *goobj2.ArchiveMember + for _, member := range pkg.ArchiveMembers { + if member.ArchiveHeader.Name == garbleSrcHeaderName { + archiveMember = &member + break + } + } + + if archiveMember == nil { + return nil + } + + osPkgPath := filepath.FromSlash(pkgPath) + pkgDebugDir := filepath.Join(envGarbleDebugDir, osPkgPath) + if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil { + return err + } + + archive := bytes.NewBuffer(archiveMember.ArchiveHeader.Data) + gzipReader, err := gzip.NewReader(archive) + if err != nil { + return err + } + defer gzipReader.Close() + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + debugFilePath := filepath.Join(pkgDebugDir, header.Name) + debugFile, err := os.Create(debugFilePath) + if err != nil { + return err + } + if _, err := io.Copy(debugFile, tarReader); err != nil { + return err + } + if err := debugFile.Close(); err != nil { + return err + } + + obfuscationTime := header.ModTime.Local() + + // Restore the actual source obfuscation time so as not to mislead the user. + if err := os.Chtimes(debugFilePath, obfuscationTime, obfuscationTime); err != nil { + return err + } + } +} + // obfuscateImports does all the necessary work to replace the import paths of // obfuscated packages with hashes. It takes the single object file and import // config passed to the linker, as well as a temporary directory to store @@ -87,6 +153,9 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string if err != nil { return "", nil, nil, fmt.Errorf("error parsing main objfile: %v", err) } + if err := extractDebugObfSrc("main", mainPkg); err != nil { + return "", nil, nil, err + } pkgs := []pkgInfo{{mainPkg, objPath, true}} privateNameMap = make(map[string]string) @@ -105,6 +174,12 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string if err := appendPrivateNameMap(pkg, privateNameMap); err != nil { return "", nil, nil, fmt.Errorf("error parsing name map %s at %s: %v", pkgPath, info.Path, err) } + // Avoiding double extraction from main object file + if objPath != info.Path { + if err := extractDebugObfSrc(pkgPath, pkg); err != nil { + return "", nil, nil, err + } + } } } @@ -120,7 +195,7 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string // skip objects that are not used by the linker, or that do not contain // any Go symbol info - if am.IsCompilerObj() || am.IsDataObj() { + if am.IsCompilerObj() || am.IsDataObj { continue } diff --git a/main.go b/main.go index 3aba7c7..8589681 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,9 @@ package main import ( + "archive/tar" "bytes" + "compress/gzip" "crypto/rand" "crypto/sha256" "encoding/base64" @@ -126,7 +128,10 @@ var ( seed []byte ) -const garbleMapHeaderName = "garble/nameMap" +const ( + garbleMapHeaderName = "garble/nameMap" + garbleSrcHeaderName = "garble/src" +) func saveListedPackages(w io.Writer, flags, patterns []string) error { args := []string{"list", "-json", "-deps", "-export"} @@ -353,15 +358,13 @@ func mainErr(args []string) error { flagDebugDir = filepath.Join(wd, flagDebugDir) } - if info, err := os.Stat(flagDebugDir); os.IsNotExist(err) { + if err := os.RemoveAll(flagDebugDir); err == nil || os.IsNotExist(err) { err := os.MkdirAll(flagDebugDir, 0o755) if err != nil { return err } - } else if err != nil { + } else { return fmt.Errorf("debugdir error: %v", err) - } else if !info.IsDir() { - return fmt.Errorf("debugdir exists, but is a file not a directory") } } @@ -427,10 +430,6 @@ func mainErr(args []string) error { "-trimpath", "-toolexec=" + execPath, } - if flagDebugDir != "" { - // TODO: don't make -debugdir force rebuilding all packages - goArgs = append(goArgs, "-a") - } if cmd == "test" { // vet is generally not useful on garbled code; keep it // disabled by default. @@ -648,15 +647,6 @@ func transformCompile(args []string) ([]string, error) { flags = flagSetValue(flags, "-trimpath", tempDir+"=>;"+trimpath) // log.Println(flags) - pkgDebugDir := "" - if envGarbleDebugDir != "" { - osPkgPath := filepath.FromSlash(pkgPath) - pkgDebugDir = filepath.Join(envGarbleDebugDir, osPkgPath) - if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil { - return nil, err - } - } - privateNameMap := make(map[string]string) existingNames := collectNames(files) packageCounter := 0 @@ -676,6 +666,13 @@ func transformCompile(args []string) ([]string, error) { files[i] = file } + obfSrcArchive := &bytes.Buffer{} + obfSrcGzipWriter := gzip.NewWriter(obfSrcArchive) + defer obfSrcGzipWriter.Close() + + obfSrcTarWriter := tar.NewWriter(obfSrcGzipWriter) + defer obfSrcTarWriter.Close() + // TODO: randomize the order and names of the files newPaths := make([]string, 0, len(files)) for i, file := range files { @@ -719,17 +716,8 @@ func transformCompile(args []string) ([]string, error) { } defer tempFile.Close() - var printWriter io.Writer = tempFile - var debugFile *os.File - if pkgDebugDir != "" { - debugFile, err = os.Create(filepath.Join(pkgDebugDir, name)) - if err != nil { - return nil, err - } - defer debugFile.Close() - - printWriter = io.MultiWriter(tempFile, debugFile) - } + obfSrc := &bytes.Buffer{} + printWriter := io.MultiWriter(tempFile, obfSrc) fileDetachedComments := detachedComments[i] if len(fileDetachedComments) > 0 { @@ -745,42 +733,57 @@ func transformCompile(args []string) ([]string, error) { if err := tempFile.Close(); err != nil { return nil, err } - debugFile.Close() // this is ok to error if no file is supplied + + if err := obfSrcTarWriter.WriteHeader(&tar.Header{ + Name: name, + Mode: 0o755, + ModTime: time.Now(), // Need for restoring obfuscation time + Size: int64(obfSrc.Len()), + }); err != nil { + return nil, err + } + if _, err := obfSrcTarWriter.Write(obfSrc.Bytes()); err != nil { + return nil, err + } newPaths = append(newPaths, tempFile.Name()) } - if len(privateNameMap) > 0 { - objPath := flagValue(flags, "-o") - deferred = append(deferred, func() error { - importCfg, err := goobj2.ParseImportCfg(buildInfo.importCfg) - if err != nil { - return err - } + objPath := flagValue(flags, "-o") + deferred = append(deferred, func() error { + importCfg, err := goobj2.ParseImportCfg(buildInfo.importCfg) + if err != nil { + return err + } - pkg, err := goobj2.Parse(objPath, pkgPath, importCfg) - if err != nil { - return err - } + pkg, err := goobj2.Parse(objPath, pkgPath, importCfg) + if err != nil { + return err + } - data, err := json.Marshal(privateNameMap) - if err != nil { - return err - } + data, err := json.Marshal(privateNameMap) + if err != nil { + return err + } - // Adding an extra archive header is safe, - // and shouldn't break other tools like the linker since our header name is unique - pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{ - ArchiveHeader: goobj2.ArchiveHeader{ - Name: garbleMapHeaderName, - Size: int64(len(data)), - Data: data, - }, - }) - - return pkg.Write(objPath) + // Adding an extra archive header is safe, + // and shouldn't break other tools like the linker since our header name is unique + pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{ + ArchiveHeader: goobj2.ArchiveHeader{ + Name: garbleMapHeaderName, + Size: int64(len(data)), + Data: data, + }, + }, goobj2.ArchiveMember{ + ArchiveHeader: goobj2.ArchiveHeader{ + Name: garbleSrcHeaderName, + Size: int64(obfSrcArchive.Len()), + Data: obfSrcArchive.Bytes(), + }, }) - } + + return pkg.Write(objPath) + }) return append(flags, newPaths...), nil } diff --git a/testdata/scripts/debugdir.txt b/testdata/scripts/debugdir.txt index 03a0c34..45dffaf 100644 --- a/testdata/scripts/debugdir.txt +++ b/testdata/scripts/debugdir.txt @@ -6,6 +6,15 @@ exists 'test1/test/main/imported/imported.go' 'test1/main/main.go' ! grep ImportedFunc $WORK/test1/main/main.go ! grep 'some comment' $WORK/test1/main/main.go +[short] stop + +# Sources from previous builds should be deleted +cp $WORK/test1/main/main.go $WORK/test1/some_file_from_prev_build.go + +garble -debugdir ./test1 build -v +! stderr 'test/main' +! exists $WORK/test1/some_file_from_prev_build.go + -- go.mod -- module test/main -- main.go --