Store obfuscated sources in object files (#158)

Now the flag "-debugdir" does not trigger a full recompilation.
Obfuscated source files are saved to object files and are extracted during linking.

go.mod

@@ -3,7 +3,7 @@ module mvdan.cc/garble
 go 1.15
 
 require (
-	github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7
+	github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e
 	github.com/google/go-cmp v0.5.2
 	github.com/rogpeppe/go-internal v1.6.3-0.20201011174404-9f985d550aa7
 	golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449

go.sum

@@ -1,5 +1,7 @@
 github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7 h1:W9kchICcGYOJQBZNn2P6j739QbY8OKskB3SyFZ/IRIU=
 github.com/Binject/debug v0.0.0-20200928131656-23d735563ce7/go.mod h1:QzgxDLY/qdKlvnbnb65eqTedhvQPbaSP2NqIbcuKvsQ=
+github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e h1:IQMhZQS5d9OUmcXgv330n31ZnD97HAZE36K7JgIjvog=
+github.com/Binject/debug v0.0.0-20201021202824-cc437dcdb16e/go.mod h1:QzgxDLY/qdKlvnbnb65eqTedhvQPbaSP2NqIbcuKvsQ=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=

import_obfuscation.go

@@ -4,12 +4,16 @@
 package main
 
 import (
+	"archive/tar"
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -62,6 +66,68 @@ func appendPrivateNameMap(pkg *goobj2.Package, nameMap map[string]string) error
 	return nil
 }
 
+// extractDebugObfSrc extracts obfuscated sources from object files if -debugdir flag is enabled.
+func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error {
+	if envGarbleDebugDir == "" {
+		return nil
+	}
+
+	var archiveMember *goobj2.ArchiveMember
+	for _, member := range pkg.ArchiveMembers {
+		if member.ArchiveHeader.Name == garbleSrcHeaderName {
+			archiveMember = &member
+			break
+		}
+	}
+
+	if archiveMember == nil {
+		return nil
+	}
+
+	osPkgPath := filepath.FromSlash(pkgPath)
+	pkgDebugDir := filepath.Join(envGarbleDebugDir, osPkgPath)
+	if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil {
+		return err
+	}
+
+	archive := bytes.NewBuffer(archiveMember.ArchiveHeader.Data)
+	gzipReader, err := gzip.NewReader(archive)
+	if err != nil {
+		return err
+	}
+	defer gzipReader.Close()
+	tarReader := tar.NewReader(gzipReader)
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		debugFilePath := filepath.Join(pkgDebugDir, header.Name)
+		debugFile, err := os.Create(debugFilePath)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(debugFile, tarReader); err != nil {
+			return err
+		}
+		if err := debugFile.Close(); err != nil {
+			return err
+		}
+
+		obfuscationTime := header.ModTime.Local()
+
+		// Restore the actual source obfuscation time so as not to mislead the user.
+		if err := os.Chtimes(debugFilePath, obfuscationTime, obfuscationTime); err != nil {
+			return err
+		}
+	}
+}
+
 // obfuscateImports does all the necessary work to replace the import paths of
 // obfuscated packages with hashes. It takes the single object file and import
 // config passed to the linker, as well as a temporary directory to store
@@ -87,6 +153,9 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string
 	if err != nil {
 		return "", nil, nil, fmt.Errorf("error parsing main objfile: %v", err)
 	}
+	if err := extractDebugObfSrc("main", mainPkg); err != nil {
+		return "", nil, nil, err
+	}
 	pkgs := []pkgInfo{{mainPkg, objPath, true}}
 
 	privateNameMap = make(map[string]string)
@@ -105,6 +174,12 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string
 			if err := appendPrivateNameMap(pkg, privateNameMap); err != nil {
 				return "", nil, nil, fmt.Errorf("error parsing name map %s at %s: %v", pkgPath, info.Path, err)
 			}
+			// Avoiding double extraction from main object file
+			if objPath != info.Path {
+				if err := extractDebugObfSrc(pkgPath, pkg); err != nil {
+					return "", nil, nil, err
+				}
+			}
 		}
 	}
 
@@ -120,7 +195,7 @@ func obfuscateImports(objPath, tempDir, importCfgPath string) (garbledObj string
 
 			// skip objects that are not used by the linker, or that do not contain
 			// any Go symbol info
-			if am.IsCompilerObj() || am.IsDataObj() {
+			if am.IsCompilerObj() || am.IsDataObj {
 				continue
 			}
 

main.go

@@ -4,7 +4,9 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
@@ -126,7 +128,10 @@ var (
 	seed []byte
 )
 
-const garbleMapHeaderName = "garble/nameMap"
+const (
+	garbleMapHeaderName = "garble/nameMap"
+	garbleSrcHeaderName = "garble/src"
+)
 
 func saveListedPackages(w io.Writer, flags, patterns []string) error {
 	args := []string{"list", "-json", "-deps", "-export"}
@@ -353,15 +358,13 @@ func mainErr(args []string) error {
 				flagDebugDir = filepath.Join(wd, flagDebugDir)
 			}
 
-			if info, err := os.Stat(flagDebugDir); os.IsNotExist(err) {
+			if err := os.RemoveAll(flagDebugDir); err == nil || os.IsNotExist(err) {
 				err := os.MkdirAll(flagDebugDir, 0o755)
 				if err != nil {
 					return err
 				}
-			} else if err != nil {
+			} else {
 				return fmt.Errorf("debugdir error: %v", err)
-			} else if !info.IsDir() {
-				return fmt.Errorf("debugdir exists, but is a file not a directory")
 			}
 		}
 
@@ -427,10 +430,6 @@ func mainErr(args []string) error {
 			"-trimpath",
 			"-toolexec=" + execPath,
 		}
-		if flagDebugDir != "" {
-			// TODO: don't make -debugdir force rebuilding all packages
-			goArgs = append(goArgs, "-a")
-		}
 		if cmd == "test" {
 			// vet is generally not useful on garbled code; keep it
 			// disabled by default.
@@ -648,15 +647,6 @@ func transformCompile(args []string) ([]string, error) {
 	flags = flagSetValue(flags, "-trimpath", tempDir+"=>;"+trimpath)
 	// log.Println(flags)
 
-	pkgDebugDir := ""
-	if envGarbleDebugDir != "" {
-		osPkgPath := filepath.FromSlash(pkgPath)
-		pkgDebugDir = filepath.Join(envGarbleDebugDir, osPkgPath)
-		if err := os.MkdirAll(pkgDebugDir, 0o755); err != nil {
-			return nil, err
-		}
-	}
-
 	privateNameMap := make(map[string]string)
 	existingNames := collectNames(files)
 	packageCounter := 0
@@ -676,6 +666,13 @@ func transformCompile(args []string) ([]string, error) {
 		files[i] = file
 	}
 
+	obfSrcArchive := &bytes.Buffer{}
+	obfSrcGzipWriter := gzip.NewWriter(obfSrcArchive)
+	defer obfSrcGzipWriter.Close()
+
+	obfSrcTarWriter := tar.NewWriter(obfSrcGzipWriter)
+	defer obfSrcTarWriter.Close()
+
 	// TODO: randomize the order and names of the files
 	newPaths := make([]string, 0, len(files))
 	for i, file := range files {
@@ -719,17 +716,8 @@ func transformCompile(args []string) ([]string, error) {
 		}
 		defer tempFile.Close()
 
-		var printWriter io.Writer = tempFile
-		var debugFile *os.File
-		if pkgDebugDir != "" {
-			debugFile, err = os.Create(filepath.Join(pkgDebugDir, name))
-			if err != nil {
-				return nil, err
-			}
-			defer debugFile.Close()
-
-			printWriter = io.MultiWriter(tempFile, debugFile)
-		}
+		obfSrc := &bytes.Buffer{}
+		printWriter := io.MultiWriter(tempFile, obfSrc)
 
 		fileDetachedComments := detachedComments[i]
 		if len(fileDetachedComments) > 0 {
@@ -745,42 +733,57 @@ func transformCompile(args []string) ([]string, error) {
 		if err := tempFile.Close(); err != nil {
 			return nil, err
 		}
-		debugFile.Close() // this is ok to error if no file is supplied
+
+		if err := obfSrcTarWriter.WriteHeader(&tar.Header{
+			Name:    name,
+			Mode:    0o755,
+			ModTime: time.Now(), // Need for restoring obfuscation time
+			Size:    int64(obfSrc.Len()),
+		}); err != nil {
+			return nil, err
+		}
+		if _, err := obfSrcTarWriter.Write(obfSrc.Bytes()); err != nil {
+			return nil, err
+		}
 
 		newPaths = append(newPaths, tempFile.Name())
 	}
 
-	if len(privateNameMap) > 0 {
-		objPath := flagValue(flags, "-o")
-		deferred = append(deferred, func() error {
-			importCfg, err := goobj2.ParseImportCfg(buildInfo.importCfg)
-			if err != nil {
-				return err
-			}
+	objPath := flagValue(flags, "-o")
+	deferred = append(deferred, func() error {
+		importCfg, err := goobj2.ParseImportCfg(buildInfo.importCfg)
+		if err != nil {
+			return err
+		}
 
-			pkg, err := goobj2.Parse(objPath, pkgPath, importCfg)
-			if err != nil {
-				return err
-			}
+		pkg, err := goobj2.Parse(objPath, pkgPath, importCfg)
+		if err != nil {
+			return err
+		}
 
-			data, err := json.Marshal(privateNameMap)
-			if err != nil {
-				return err
-			}
+		data, err := json.Marshal(privateNameMap)
+		if err != nil {
+			return err
+		}
 
-			// Adding an extra archive header is safe,
-			// and shouldn't break other tools like the linker since our header name is unique
-			pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{
-				ArchiveHeader: goobj2.ArchiveHeader{
-					Name: garbleMapHeaderName,
-					Size: int64(len(data)),
-					Data: data,
-				},
-			})
-
-			return pkg.Write(objPath)
+		// Adding an extra archive header is safe,
+		// and shouldn't break other tools like the linker since our header name is unique
+		pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{
+			ArchiveHeader: goobj2.ArchiveHeader{
+				Name: garbleMapHeaderName,
+				Size: int64(len(data)),
+				Data: data,
+			},
+		}, goobj2.ArchiveMember{
+			ArchiveHeader: goobj2.ArchiveHeader{
+				Name: garbleSrcHeaderName,
+				Size: int64(obfSrcArchive.Len()),
+				Data: obfSrcArchive.Bytes(),
+			},
 		})
-	}
+
+		return pkg.Write(objPath)
+	})
 
 	return append(flags, newPaths...), nil
 }

testdata/scripts/debugdir.txt

@@ -6,6 +6,15 @@ exists 'test1/test/main/imported/imported.go' 'test1/main/main.go'
 ! grep ImportedFunc $WORK/test1/main/main.go
 ! grep 'some comment' $WORK/test1/main/main.go
 
+[short] stop
+
+# Sources from previous builds should be deleted
+cp $WORK/test1/main/main.go $WORK/test1/some_file_from_prev_build.go
+
+garble -debugdir ./test1 build -v
+! stderr 'test/main'
+! exists $WORK/test1/some_file_from_prev_build.go
+
 -- go.mod --
 module test/main
 -- main.go --