Fix token scrambling to preserve type hints and magic constants

Dockerfile.test-symbols

@@ -6,22 +6,22 @@ FROM golang:1.21-bookworm AS builder
 WORKDIR /build
 
 # Copy scrambler source
-COPY encrypted-execution/tools/scrambler/*.go encrypted-execution/tools/scrambler/go.mod ./
+COPY encrypted-execution/src/scrambler/*.go encrypted-execution/src/scrambler/go.mod ./
 
 # Build scrambler
 RUN go build -o php-scrambler .
 
 # Runtime stage
 FROM debian:bookworm-slim
 
-# Install Python for transformation script
-RUN apt-get update && apt-get install -y python3 && rm -rf /var/lib/apt/lists/*
+# Install PHP for transformation script
+RUN apt-get update && apt-get install -y php-cli && rm -rf /var/lib/apt/lists/*
 
 # Copy built scrambler
 COPY --from=builder /build/php-scrambler /usr/local/bin/
 
 # Copy token-aware transformation script
-COPY encrypted-execution/tools/scrambler/token-aware-transformer.php /usr/local/bin/
+COPY encrypted-execution/src/transformer/token-aware-transformer.php /usr/local/bin/
 RUN chmod +x /usr/local/bin/token-aware-transformer.php
 
 # Create test directory

encrypted-execution/src/scrambler/dictionaryHandler.go

@@ -33,6 +33,19 @@ var KeywordsRegex = regexp.MustCompile( //REGEX found as user @martindilling com
 		"(break|list|(x)?or|var|while)|" +
 		"(string|object|list|int(eger)?|real|float|[^_]AND|[^(R|_|F)(X)?)](X)?OR))[^a-zA-Z0-9]")
 
+// PHP magic constants that must NEVER be scrambled
+// These are compile-time constants that PHP replaces with values
+var MagicConstants = map[string]bool{
+	"__LINE__":     true,
+	"__FILE__":     true,
+	"__DIR__":      true,
+	"__FUNCTION__": true,
+	"__CLASS__":    true,
+	"__TRAIT__":    true,
+	"__METHOD__":   true,
+	"__NAMESPACE__": true,
+}
+
 var EEWords = make(map[string]string)
 var SpecialChar = make(map[string]string)
 var PreMadeDict = false
@@ -50,6 +63,12 @@ func InitEEWords(filename string) {
 
 func AddToEEWords(key string) bool {
 	var ok bool
+
+	// CRITICAL: Never scramble magic constants
+	if MagicConstants[key] {
+		return false
+	}
+
 	if _, ok = EEWords[key]; ok {
 		return false
 	} else {
@@ -112,8 +131,38 @@ var CharMatches = []string{}
 
 var CharStrRegex = regexp.MustCompile("(\")[^\\w\"]{2,}[ \"]")
 
-var symbolChars = [...]string{"(", ")", "]", "-", "~", "^", "&", "@", "!", "|", "+", ":", "=", ",", "%"}
-var specialChars = []string{"(", ")", "]"}
+// Expanded symbol set for comprehensive scrambling
+// Note: Some symbols are excluded due to tokenization issues:
+// - '[' causes variable parsing issues in strings
+// - '.' causes decimal number issues
+// - '>', '<', '?' cause PHP tag issues
+// - '$' causes variable issues
+// - '/', '*' cause comment issues
+// SYMBOL SCRAMBLING CONSTRAINTS:
+//
+// Brackets are EXCLUDED - hard-coded in PHP's lexer:
+// - ( ) for function calls and grouping
+// - [ ] for array access: $array['key']
+// - { } for string interpolation: "text {$var} more"
+//
+// Multi-character operators are EXCLUDED - recognized as single tokens:
+// - = forms: ==, ===, !=, !==, <=, >=, =>, +=, -=, *=, /=, %=, .=, etc.
+// - & forms: &&, &=
+// - | forms: ||, |=
+// - ! forms: !=, !==
+// - - forms: --, -=, ->
+// - + forms: ++, +=
+// - ^ forms: ^=
+// - % forms: %=
+//
+// SAFE symbols (standalone, don't form multi-char operators):
+var symbolChars = [...]string{
+	";",   // Statement terminator
+	",",   // Comma separator
+	"@",   // Error suppression
+	"~",   // Bitwise NOT (standalone operator)
+}
+var specialChars = []string{";"}
 
 func shuffle() []string {
 	r := rand.New(rand.NewSource(time.Now().Unix()))
@@ -128,18 +177,21 @@ func shuffle() []string {
 func InitChar() {
 	// create Char Matchers
 	addCharMatches(specialChars, []string{"\"", "'"})
-	addCharMatches([]string{"-", "~", "^", "&", "@", "!", "|", "+", ":", "=", ",", "%"}, []string{"'"})
+	// Add matches for other symbols (only in single quotes to be safer)
+	addCharMatches([]string{"-", "+", "=", "%", "~", "^", "&", "@", "!", "|", ":", ","}, []string{"'"})
 
 	if !PreMadeDict {
 		permutationGen()
 	}
 
 	for _, char := range specialChars {
 		out := EEWords[char]
-		if char == "]" {
-			char = "\\]"
+		// Escape special regex characters
+		escaped := char
+		if char == "]" || char == "[" || char == "{" || char == "}" {
+			escaped = "\\" + char
 		}
-		SpecialChar[char] = out
+		SpecialChar[escaped] = out
 	}
 }
 
@@ -158,9 +210,7 @@ func permutationGen() {
 		EEWords[char], permutation = permutation[0], permutation[1:]
 	}
 
-	if EEWords["("] == "]" || EEWords[")"] == "]" {
-		permutationGen()
-	}
+	// No bracket pairing constraints needed since brackets are excluded from scrambling
 	return
 }
 

encrypted-execution/src/scrambler/go.mod

@@ -0,0 +1,5 @@
+module github.com/encrypted-execution/php-scrambler
+
+go 1.21
+
+require ()

encrypted-execution/src/scrambler/scrambler.go

@@ -62,7 +62,10 @@ func checkTokens(lexFile string) {
 	// We need to do reverse mapping in lex file: if '(' -> ')' in source,
 	// then ')' -> '(' in lex file so it interprets correctly
 
-	const tokensPattern = "TOKENS [;:,.|^&+-/*=%!~$<>?@]"
+	// Original TOKENS pattern from PHP source
+	const originalTokensPattern = "TOKENS [;:,.|^&+-/*=%!~$<>?@]"
+	// Expanded pattern with our additional symbols (- at end to avoid range issues)
+	const expandedTokensPattern = "TOKENS [;:,.|^&+/*=%!~$<>?@\\[\\]{}()-]"
 	const varOffsetPrefix = "<ST_VAR_OFFSET>{TOKENS}|"
 
 	lines := strings.Split(string(file), "\n")
@@ -72,38 +75,17 @@ func checkTokens(lexFile string) {
 	for i, line := range lines {
 		trimmed := strings.TrimSpace(line)
 
-		// Only match exact TOKENS macro definition line (not C code)
-		if trimmed == tokensPattern && !tokensReplaced {
-			// Start with original character set
-			outLine := tokensPattern
-			// Replace each scrambled char with its original (reverse mapping)
-			for key, val := range SpecialChar {
-				// key is the escaped version (like "\\]"), val is what it maps to
-				// We want to replace val with the unescaped key in the character class
-				unescapedKey := strings.Replace(key, "\\", "", -1)
-				outLine = strings.Replace(outLine, val, unescapedKey, 1)
-			}
-			lines[i] = outLine
+		// Match either original or expanded TOKENS pattern
+		if (trimmed == originalTokensPattern || trimmed == expandedTokensPattern) && !tokensReplaced {
+			// Simply expand to include all our symbols without reverse mapping
+			// The scanLines() function will handle the actual symbol transformation
+			lines[i] = expandedTokensPattern
 			tokensReplaced = true
 			fmt.Printf("Replaced TOKENS definition at line %d\n", i+1)
 		} else if strings.Contains(trimmed, varOffsetPrefix) && strings.Contains(trimmed, "[[") && !varOffsetReplaced {
-			// Handle the ST_VAR_OFFSET line - original format varies by PHP version
-			// Find the character class after the | and before the ] {
-			start := strings.Index(line, "|")
-			end := strings.Index(line[start:], "]")
-			if start != -1 && end != -1 {
-				// Extract the character class
-				charClass := line[start+1 : start+end]
-				// Apply reverse mapping to special characters
-				for key, val := range SpecialChar {
-					unescapedKey := strings.Replace(key, "\\", "", -1)
-					charClass = strings.Replace(charClass, val, unescapedKey, 1)
-				}
-				// Reconstruct the line
-				lines[i] = line[:start+1] + charClass + line[start+end:]
-				varOffsetReplaced = true
-				fmt.Printf("Replaced ST_VAR_OFFSET TOKENS usage at line %d\n", i+1)
-			}
+			// Handle the ST_VAR_OFFSET line - just note it, scanLines will transform it
+			varOffsetReplaced = true
+			fmt.Printf("Found ST_VAR_OFFSET TOKENS usage at line %d (will be transformed by scanLines)\n", i+1)
 		}
 	}
 
@@ -174,6 +156,16 @@ func substituteWordsInString(line string) string {
 		matchedRegex = strings.TrimSuffix(strings.TrimPrefix(matchedRegex, prefix), suffix)
 		key := strings.TrimPrefix(matchedRegex, "\"")
 
+		// CRITICAL: Skip magic constants - they must never be scrambled
+		if MagicConstants[key] || MagicConstants[strings.ToUpper(key)] {
+			// Don't scramble this token, move to next match
+			matchedRegex = KeywordsRegex.FindString(line[index[1]:])
+			if matchedRegex == "" {
+				break
+			}
+			continue
+		}
+
 		if _, ok := GetScrambled(key); ok || PreMadeDict {
 			key, _ = GetScrambled(key)
 		} else {