Merge d8d33f17b5 into 8ee4c91196

internal/asthelper/asthelper.go

@@ -59,6 +59,22 @@ func LambdaCall(resultType ast.Expr, block *ast.BlockStmt) *ast.CallExpr {
 	return CallExpr(funcLit)
 }
 
+// LambdaCallParams "func(params) resultType {block}(args)"
+func LambdaCallParams(params *ast.FieldList, resultType ast.Expr, block *ast.BlockStmt, args []ast.Expr) *ast.CallExpr {
+	funcLit := &ast.FuncLit{
+		Type: &ast.FuncType{
+			Params: params,
+			Results: &ast.FieldList{
+				List: []*ast.Field{
+					{Type: resultType},
+				},
+			},
+		},
+		Body: block,
+	}
+	return CallExpr(funcLit, args...)
+}
+
 // ReturnStmt "return result"
 func ReturnStmt(results ...ast.Expr) *ast.ReturnStmt {
 	return &ast.ReturnStmt{

internal/literals/literals.go

@@ -110,7 +110,29 @@ func Obfuscate(rand *mathrand.Rand, file *ast.File, info *types.Info, linkString
 		return true
 	}
 
-	return astutil.Apply(file, pre, post).(*ast.File)
+	newFile := astutil.Apply(file, pre, post).(*ast.File)
+	if obfRand.globalKeysUsed {
+		// Generate a global variable containing global keys
+		// var __garble_global_keys_%d = [...]uint64{ <global keys> }
+		elts := make([]ast.Expr, len(obfRand.globalKeys))
+		for i, key := range obfRand.globalKeys {
+			elts[i] = &ast.BasicLit{Kind: token.INT, Value: fmt.Sprint(key)}
+		}
+		newFile.Decls = append(newFile.Decls, &ast.GenDecl{
+			Tok: token.VAR,
+			Specs: []ast.Spec{&ast.ValueSpec{
+				Names: []*ast.Ident{ast.NewIdent(obfRand.globalKeysVarName)},
+				Values: []ast.Expr{&ast.CompositeLit{
+					Type: &ast.ArrayType{
+						Len: ah.IntLit(len(obfRand.globalKeys)),
+						Elt: ast.NewIdent("uint64"),
+					},
+					Elts: elts,
+				}},
+			}},
+		})
+	}
+	return newFile
 }
 
 // handleCompositeLiteral checks if the input node is []byte or [...]byte and
@@ -213,34 +235,43 @@ func withPos(node ast.Node, pos token.Pos) ast.Node {
 
 func obfuscateString(obfRand *obfRand, data string) *ast.CallExpr {
 	obf := getNextObfuscator(obfRand, len(data))
-	block := obf.obfuscate(obfRand.Rand, []byte(data))
+
+	extKeys := randExtKeys(obfRand.Rand)
+	block := obf.obfuscate(obfRand.Rand, []byte(data), extKeys)
+	params, args := extKeysToParams(obfRand, extKeys)
 
 	block.List = append(block.List, ah.ReturnStmt(ah.CallExpr(ast.NewIdent("string"), ast.NewIdent("data"))))
 
-	return ah.LambdaCall(ast.NewIdent("string"), block)
+	return ah.LambdaCallParams(params, ast.NewIdent("string"), block, args)
 }
 
 func obfuscateByteSlice(obfRand *obfRand, isPointer bool, data []byte) *ast.CallExpr {
 	obf := getNextObfuscator(obfRand, len(data))
-	block := obf.obfuscate(obfRand.Rand, data)
+
+	extKeys := randExtKeys(obfRand.Rand)
+	block := obf.obfuscate(obfRand.Rand, data, extKeys)
+	params, args := extKeysToParams(obfRand, extKeys)
 
 	if isPointer {
 		block.List = append(block.List, ah.ReturnStmt(&ast.UnaryExpr{
 			Op: token.AND,
 			X:  ast.NewIdent("data"),
 		}))
-		return ah.LambdaCall(&ast.StarExpr{
+		return ah.LambdaCallParams(params, &ast.StarExpr{
 			X: &ast.ArrayType{Elt: ast.NewIdent("byte")},
-		}, block)
+		}, block, args)
 	}
 
 	block.List = append(block.List, ah.ReturnStmt(ast.NewIdent("data")))
-	return ah.LambdaCall(&ast.ArrayType{Elt: ast.NewIdent("byte")}, block)
+	return ah.LambdaCallParams(params, &ast.ArrayType{Elt: ast.NewIdent("byte")}, block, args)
 }
 
 func obfuscateByteArray(obfRand *obfRand, isPointer bool, data []byte, length int64) *ast.CallExpr {
 	obf := getNextObfuscator(obfRand, len(data))
-	block := obf.obfuscate(obfRand.Rand, data)
+
+	extKeys := randExtKeys(obfRand.Rand)
+	block := obf.obfuscate(obfRand.Rand, data, extKeys)
+	params, args := extKeysToParams(obfRand, extKeys)
 
 	arrayType := &ast.ArrayType{
 		Len: ah.IntLit(int(length)),
@@ -280,10 +311,10 @@ func obfuscateByteArray(obfRand *obfRand, isPointer bool, data []byte, length in
 	block.List = append(block.List, sliceToArray...)
 
 	if isPointer {
-		return ah.LambdaCall(&ast.StarExpr{X: arrayType}, block)
+		return ah.LambdaCallParams(params, &ast.StarExpr{X: arrayType}, block, args)
 	}
 
-	return ah.LambdaCall(arrayType, block)
+	return ah.LambdaCallParams(params, arrayType, block, args)
 }
 
 func getNextObfuscator(obfRand *obfRand, size int) obfuscator {

internal/literals/obfuscators.go

@@ -7,12 +7,54 @@ import (
 	"fmt"
 	"go/ast"
 	"go/token"
+	"math"
 	mathrand "math/rand"
+	ah "mvdan.cc/garble/internal/asthelper"
+	"slices"
 )
 
+// extKeyRarity probability of using an external key.
+// Larger value, greater probability of using an external key.
+// Must be between 0 and 1
+type extKeyRarity float32
+
+const (
+	rareRarity   extKeyRarity = 0.4
+	normalRarity extKeyRarity = 0.6
+	commonRarity extKeyRarity = 0.8
+)
+
+func (r extKeyRarity) Try(rand *mathrand.Rand) bool {
+	return rand.Float32() < float32(r)
+}
+
+// extKey contains all information about the external key
+type extKey struct {
+	name, typ string
+	value     uint64
+	bits      int
+	refs      int
+}
+
+func (k *extKey) Type() *ast.Ident {
+	return ast.NewIdent(k.typ)
+}
+
+func (k *extKey) Name() *ast.Ident {
+	return ast.NewIdent(k.name)
+}
+
+func (k *extKey) AddRef() {
+	k.refs++
+}
+
+func (k *extKey) IsUsed() bool {
+	return k.refs > 0
+}
+
 // obfuscator takes a byte slice and converts it to a ast.BlockStmt
 type obfuscator interface {
-	obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt
+	obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt
 }
 
 var (
@@ -79,9 +121,147 @@ func operatorToReversedBinaryExpr(t token.Token, x, y ast.Expr) *ast.BinaryExpr
 	return expr
 }
 
+const (
+	// minExtKeyCount is minimum number of external keys for one lambda call
+	minExtKeyCount = 2
+	// maxExtKeyCount is maximum number of external keys for one lambda call
+	maxExtKeyCount = 6
+
+	// minByteSliceExtKeyOps minimum number of operations with external keys for one byte slice
+	minByteSliceExtKeyOps = 2
+	// maxByteSliceExtKeyOps maximum number of operations with external keys for one byte slice
+	maxByteSliceExtKeyOps = 12
+)
+
+// extKeyRanges contains a list of different ranges of random numbers for external keys
+// Different types and bitnesses will increase the chance of changing patterns
+var extKeyRanges = []struct {
+	typ  string
+	max  uint64
+	bits int
+}{
+	{"uint8", math.MaxUint8, 8},
+	{"uint16", math.MaxUint16, 16},
+	{"uint32", math.MaxUint32, 32},
+	{"uint64", math.MaxUint64, 64},
+}
+
+func randExtKey(obfRand *mathrand.Rand, idx int) *extKey {
+	r := extKeyRanges[obfRand.Intn(len(extKeyRanges))]
+	return &extKey{
+		name:  fmt.Sprintf("__garble_ext_key_%d", idx),
+		typ:   r.typ,
+		value: obfRand.Uint64() & r.max,
+		bits:  r.bits,
+	}
+}
+
+func randExtKeys(obfRand *mathrand.Rand) []*extKey {
+	count := minExtKeyCount + obfRand.Intn(maxExtKeyCount-minExtKeyCount)
+
+	keys := make([]*extKey, count)
+	for i := 0; i < count; i++ {
+		keys[i] = randExtKey(obfRand, i)
+	}
+
+	return keys
+}
+
+func extKeysToParams(obfRand *obfRand, keys []*extKey) (params *ast.FieldList, args []ast.Expr) {
+	params = &ast.FieldList{}
+	for _, key := range keys {
+		name := key.Name()
+		if !key.IsUsed() {
+			name.Name = "_"
+		}
+		params.List = append(params.List, &ast.Field{
+			Names: []*ast.Ident{name},
+			Type:  key.Type(),
+		})
+
+		if rareRarity.Try(obfRand.Rand) {
+			args = append(args, obfRand.scrambleUsingGlobalKey(key))
+		} else {
+			args = append(args, &ast.BasicLit{
+				Kind:  token.INT,
+				Value: fmt.Sprint(key.value),
+			})
+		}
+	}
+	return
+}
+
+// dataToByteSliceWithExtKeys scramble and turns a byte slice into an AST expression like:
+//
+//	func() []byte {
+//		data := []byte("<data>")
+//		data[<index>] = data[<index>] <random operator> byte(<external key> >> <random shift>) // repeated random times
+//		return data
+//	}()
+func dataToByteSliceWithExtKeys(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) ast.Expr {
+	extKeyOpCount := minByteSliceExtKeyOps + obfRand.Intn(maxByteSliceExtKeyOps-minByteSliceExtKeyOps)
+
+	var stmts []ast.Stmt
+	for i := 0; i < extKeyOpCount; i++ {
+		key := extKeys[obfRand.Intn(len(extKeys))]
+		key.AddRef()
+
+		idx, op, b := obfRand.Intn(len(data)), randOperator(obfRand), obfRand.Intn(key.bits/8)
+		data[idx] = evalOperator(op, data[idx], byte(key.value>>(b*8)))
+		stmts = append(stmts, &ast.AssignStmt{
+			Lhs: []ast.Expr{ah.IndexExpr("data", ah.IntLit(idx))},
+			Tok: token.ASSIGN,
+			Rhs: []ast.Expr{
+				operatorToReversedBinaryExpr(op,
+					ah.IndexExpr("data", ah.IntLit(idx)),
+					ah.CallExprByName("byte", &ast.BinaryExpr{
+						X:  key.Name(),
+						Op: token.SHR,
+						Y:  ah.IntLit(b * 8),
+					}),
+				),
+			},
+		})
+	}
+
+	// External keys can be applied several times to the same array element,
+	// and it is important to invert the order of execution to correctly restore the original value
+	slices.Reverse(stmts)
+
+	stmts = append([]ast.Stmt{ah.AssignDefineStmt(ast.NewIdent("data"), ah.DataToByteSlice(data))}, append(stmts, ah.ReturnStmt(ast.NewIdent("data")))...)
+	return ah.LambdaCall(&ast.ArrayType{Elt: ast.NewIdent("byte")}, ah.BlockStmt(stmts...))
+}
+
+// dataToByteSliceWithExtKeys scramble and turns a byte into an AST expression like:
+//
+//	byte(<obfuscated value>) <random operator> byte(<external key> >> <random shift>)
+func byteLitWithExtKey(obfRand *mathrand.Rand, val byte, extKeys []*extKey, rarity extKeyRarity) ast.Expr {
+	if !rarity.Try(obfRand) {
+		return ah.IntLit(int(val))
+	}
+
+	key := extKeys[obfRand.Intn(len(extKeys))]
+	key.AddRef()
+
+	op, b := randOperator(obfRand), obfRand.Intn(key.bits/8)
+	newVal := evalOperator(op, val, byte(key.value>>(b*8)))
+	return operatorToReversedBinaryExpr(op,
+		ah.CallExprByName("byte", ah.IntLit(int(newVal))),
+		ah.CallExprByName("byte", &ast.BinaryExpr{
+			X:  key.Name(),
+			Op: token.SHR,
+			Y:  ah.IntLit(b * 8),
+		}),
+	)
+}
+
 type obfRand struct {
 	*mathrand.Rand
 	testObfuscator obfuscator
+
+	globalKeysVarName string
+	globalKeys        []uint64
+	globalKeysUsed    bool
 }
 
 func (r *obfRand) nextObfuscator() obfuscator {
@@ -98,7 +278,26 @@ func (r *obfRand) nextLinearTimeObfuscator() obfuscator {
 	return Obfuscators[r.Intn(len(LinearTimeObfuscators))]
 }
 
+func (r *obfRand) scrambleUsingGlobalKey(k *extKey) ast.Expr {
+	r.globalKeysUsed = true
+
+	extKeyIdx := r.Intn(len(r.globalKeys))
+	return &ast.BinaryExpr{
+		X:  ah.CallExprByName(k.typ, ah.IndexExpr(r.globalKeysVarName, ah.IntLit(extKeyIdx))),
+		Op: token.XOR,
+		Y: &ast.BasicLit{
+			Kind: token.INT,
+			// To avoid an overflow error at compile time, truncate global key to external key bitness
+			Value: fmt.Sprint(k.value ^ (r.globalKeys[extKeyIdx] & ((1 << k.bits) - 1))),
+		},
+	}
+}
+
 func newObfRand(rand *mathrand.Rand, file *ast.File) *obfRand {
 	testObf := testPkgToObfuscatorMap[file.Name.Name]
-	return &obfRand{rand, testObf}
+	globalKeys := make([]uint64, minExtKeyCount+rand.Intn(maxExtKeyCount-minExtKeyCount))
+	for i := 0; i < len(globalKeys); i++ {
+		globalKeys[i] = rand.Uint64()
+	}
+	return &obfRand{rand, testObf, fmt.Sprintf("__garble_global_keys_%d", rand.Uint64()), globalKeys, false}
 }

internal/literals/seed.go

@@ -16,30 +16,29 @@ type seed struct{}
 // check that the obfuscator interface is implemented
 var _ obfuscator = seed{}
 
-func (seed) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
+func (seed) obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt {
 	seed := byte(obfRand.Uint32())
 	originalSeed := seed
 
 	op := randOperator(obfRand)
-
 	var callExpr *ast.CallExpr
 	for i, b := range data {
 		encB := evalOperator(op, b, seed)
 		seed += encB
 
 		if i == 0 {
-			callExpr = ah.CallExpr(ast.NewIdent("fnc"), ah.IntLit(int(encB)))
+			callExpr = ah.CallExpr(ast.NewIdent("fnc"), byteLitWithExtKey(obfRand, encB, extKeys, commonRarity))
 			continue
 		}
 
-		callExpr = ah.CallExpr(callExpr, ah.IntLit(int(encB)))
+		callExpr = ah.CallExpr(callExpr, byteLitWithExtKey(obfRand, encB, extKeys, rareRarity))
 	}
 
 	return ah.BlockStmt(
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("seed")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.CallExpr(ast.NewIdent("byte"), ah.IntLit(int(originalSeed)))},
+			Rhs: []ast.Expr{ah.CallExprByName("byte", byteLitWithExtKey(obfRand, originalSeed, extKeys, commonRarity))},
 		},
 		&ast.DeclStmt{
 			Decl: &ast.GenDecl{

internal/literals/shuffle.go

@@ -16,7 +16,7 @@ type shuffle struct{}
 // check that the obfuscator interface is implemented
 var _ obfuscator = shuffle{}
 
-func (shuffle) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
+func (shuffle) obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt {
 	key := make([]byte, len(data))
 	obfRand.Read(key)
 
@@ -69,12 +69,12 @@ func (shuffle) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("fullData")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.DataToByteSlice(shuffledFullData)},
+			Rhs: []ast.Expr{dataToByteSliceWithExtKeys(obfRand, shuffledFullData, extKeys)},
 		},
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("idxKey")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.DataToByteSlice(idxKey)},
+			Rhs: []ast.Expr{dataToByteSliceWithExtKeys(obfRand, idxKey, extKeys)},
 		},
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("data")},

internal/literals/simple.go

@@ -7,7 +7,6 @@ import (
 	"go/ast"
 	"go/token"
 	mathrand "math/rand"
-
 	ah "mvdan.cc/garble/internal/asthelper"
 )
 
@@ -16,7 +15,7 @@ type simple struct{}
 // check that the obfuscator interface is implemented
 var _ obfuscator = simple{}
 
-func (simple) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
+func (simple) obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt {
 	key := make([]byte, len(data))
 	obfRand.Read(key)
 
@@ -29,12 +28,12 @@ func (simple) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("key")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.DataToByteSlice(key)},
+			Rhs: []ast.Expr{dataToByteSliceWithExtKeys(obfRand, key, extKeys)},
 		},
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("data")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.DataToByteSlice(data)},
+			Rhs: []ast.Expr{dataToByteSliceWithExtKeys(obfRand, data, extKeys)},
 		},
 		&ast.RangeStmt{
 			Key:   ast.NewIdent("i"),

internal/literals/split.go

@@ -66,7 +66,7 @@ func encryptChunks(chunks [][]byte, op token.Token, key byte) {
 	}
 }
 
-func (split) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
+func (split) obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt {
 	var chunks [][]byte
 	// Short arrays should be divided into single-byte fragments
 	if len(data)/maxChunkSize < minCaseCount {
@@ -131,10 +131,10 @@ func (split) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 		}
 
 		if len(chunk) != 1 {
-			appendCallExpr.Args = append(appendCallExpr.Args, ah.StringLit(string(chunk)))
+			appendCallExpr.Args = append(appendCallExpr.Args, dataToByteSliceWithExtKeys(obfRand, chunk, extKeys))
 			appendCallExpr.Ellipsis = 1
 		} else {
-			appendCallExpr.Args = append(appendCallExpr.Args, ah.IntLit(int(chunk[0])))
+			appendCallExpr.Args = append(appendCallExpr.Args, byteLitWithExtKey(obfRand, chunk[0], extKeys, rareRarity))
 		}
 
 		switchCases = append(switchCases, &ast.CaseClause{
@@ -168,7 +168,7 @@ func (split) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("decryptKey")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.IntLit(int(decryptKeyInitial))},
+			Rhs: []ast.Expr{ah.CallExprByName("int", byteLitWithExtKey(obfRand, decryptKeyInitial, extKeys, normalRarity))},
 		},
 		&ast.ForStmt{
 			Init: &ast.AssignStmt{

internal/literals/swap.go

@@ -58,7 +58,7 @@ func generateSwapCount(obfRand *mathrand.Rand, dataLen int) int {
 	return swapCount
 }
 
-func (swap) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
+func (swap) obfuscate(obfRand *mathrand.Rand, data []byte, extKeys []*extKey) *ast.BlockStmt {
 	swapCount := generateSwapCount(obfRand, len(data))
 	shiftKey := byte(obfRand.Uint32())
 
@@ -76,7 +76,7 @@ func (swap) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("data")},
 			Tok: token.DEFINE,
-			Rhs: []ast.Expr{ah.DataToByteSlice(data)},
+			Rhs: []ast.Expr{dataToByteSliceWithExtKeys(obfRand, data, extKeys)},
 		},
 		&ast.AssignStmt{
 			Lhs: []ast.Expr{ast.NewIdent("positions")},
@@ -118,7 +118,7 @@ func (swap) obfuscate(obfRand *mathrand.Rand, data []byte) *ast.BlockStmt {
 							}),
 						},
 						Op: token.ADD,
-						Y:  ah.IntLit(int(shiftKey)),
+						Y:  byteLitWithExtKey(obfRand, shiftKey, extKeys, commonRarity),
 					}},
 				},
 				&ast.AssignStmt{

testdata/script/literals.txtar

@@ -53,6 +53,12 @@ grep '^(\s+)?\w+ = .*\bappend\(\w+,(\s+\w+\[\d+\^\s.+\][\^\-+]\w+\[\d+\^\s.+\],?
 # XorSeed obfuscator. Detect type decFunc func(byte) decFunc
 grep '^\s+type \w+ func\(byte\) \w+$' debug1/test/main/extra_literals.go
 
+# Check external keys
+grep '__garble_ext_key_' debug1/test/main/extra_literals.go
+
+# Check global keys
+grep '__garble_global_keys_' debug1/test/main/extra_literals.go
+
 # Finally, sanity check that we can build all of std with -literals.
 # Analogous to gogarble.txt.
 exec garble -literals build std