Add HexLiteralTransformer

Signed-off-by: Graham <gpe@openrs2.dev>

deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscatorModule.kt

@@ -13,6 +13,7 @@ import dev.openrs2.deob.ast.transform.ComplementTransformer
 import dev.openrs2.deob.ast.transform.EncloseTransformer
 import dev.openrs2.deob.ast.transform.ForLoopConditionTransformer
 import dev.openrs2.deob.ast.transform.GlTransformer
+import dev.openrs2.deob.ast.transform.HexLiteralTransformer
 import dev.openrs2.deob.ast.transform.IdentityTransformer
 import dev.openrs2.deob.ast.transform.IfElseTransformer
 import dev.openrs2.deob.ast.transform.IncrementTransformer
@@ -41,6 +42,7 @@ object AstDeobfuscatorModule : AbstractModule() {
         binder.addBinding().to(AddSubTransformer::class.java)
         binder.addBinding().to(IdentityTransformer::class.java)
         binder.addBinding().to(BitMaskTransformer::class.java)
+        binder.addBinding().to(HexLiteralTransformer::class.java)
         binder.addBinding().to(ValueOfTransformer::class.java)
         binder.addBinding().to(NewInstanceTransformer::class.java)
         binder.addBinding().to(IncrementTransformer::class.java)

deob-ast/src/main/java/dev/openrs2/deob/ast/transform/GlTransformer.kt

@@ -11,7 +11,6 @@ import com.github.javaparser.ast.body.VariableDeclarator
 import com.github.javaparser.ast.expr.BinaryExpr
 import com.github.javaparser.ast.expr.Expression
 import com.github.javaparser.ast.expr.FieldAccessExpr
-import com.github.javaparser.ast.expr.IntegerLiteralExpr
 import com.github.javaparser.ast.expr.MethodCallExpr
 import com.github.javaparser.ast.expr.NameExpr
 import com.github.javaparser.ast.expr.SimpleName
@@ -26,6 +25,7 @@ import dev.openrs2.deob.ast.gl.GlEnum
 import dev.openrs2.deob.ast.gl.GlParameter
 import dev.openrs2.deob.ast.gl.GlRegistry
 import dev.openrs2.deob.ast.util.checkedAsInt
+import dev.openrs2.deob.ast.util.toHexLiteralExpr
 import dev.openrs2.deob.ast.util.walk
 import javax.inject.Inject
 import javax.inject.Singleton
@@ -229,10 +229,6 @@ class GlTransformer @Inject constructor(private val registry: GlRegistry) : Tran
         )
     }
 
-    private fun Int.toHexLiteralExpr(): IntegerLiteralExpr {
-        return IntegerLiteralExpr("0x${Integer.toUnsignedString(this, 16).toUpperCase()}")
-    }
-
     private fun transformExpr(
         unit: CompilationUnit,
         command: GlCommand,

deob-ast/src/main/java/dev/openrs2/deob/ast/transform/HexLiteralTransformer.kt

@@ -0,0 +1,48 @@
+package dev.openrs2.deob.ast.transform
+
+import com.github.javaparser.ast.CompilationUnit
+import com.github.javaparser.ast.expr.BinaryExpr
+import com.github.javaparser.ast.expr.Expression
+import com.github.javaparser.ast.expr.IntegerLiteralExpr
+import com.github.javaparser.ast.expr.LongLiteralExpr
+import dev.openrs2.deob.ast.Library
+import dev.openrs2.deob.ast.LibraryGroup
+import dev.openrs2.deob.ast.util.checkedAsInt
+import dev.openrs2.deob.ast.util.checkedAsLong
+import dev.openrs2.deob.ast.util.toHexLiteralExpr
+import dev.openrs2.deob.ast.util.walk
+
+class HexLiteralTransformer : Transformer() {
+    override fun transformUnit(group: LibraryGroup, library: Library, unit: CompilationUnit) {
+        unit.walk { expr: BinaryExpr ->
+            if (expr.operator in SHIFT_OPS) {
+                convertToHex(expr.left)
+            }
+
+            if (expr.operator in BITWISE_OPS) {
+                convertToHex(expr.left)
+                convertToHex(expr.right)
+            }
+        }
+    }
+
+    private fun convertToHex(expr: Expression) {
+        when (expr) {
+            is IntegerLiteralExpr -> expr.replace(expr.checkedAsInt().toHexLiteralExpr())
+            is LongLiteralExpr -> expr.replace(expr.checkedAsLong().toHexLiteralExpr())
+        }
+    }
+
+    private companion object {
+        private val SHIFT_OPS = setOf(
+            BinaryExpr.Operator.LEFT_SHIFT,
+            BinaryExpr.Operator.SIGNED_RIGHT_SHIFT,
+            BinaryExpr.Operator.UNSIGNED_RIGHT_SHIFT
+        )
+        private val BITWISE_OPS = setOf(
+            BinaryExpr.Operator.BINARY_AND,
+            BinaryExpr.Operator.BINARY_OR,
+            BinaryExpr.Operator.XOR
+        )
+    }
+}

deob-ast/src/main/java/dev/openrs2/deob/ast/util/ExprUtils.kt

@@ -15,6 +15,10 @@ fun IntegerLiteralExpr.checkedAsInt(): Int {
     return n
 }
 
+fun Int.toHexLiteralExpr(): IntegerLiteralExpr {
+    return IntegerLiteralExpr("0x${Integer.toUnsignedString(this, 16).toUpperCase()}")
+}
+
 fun LongLiteralExpr.checkedAsLong(): Long {
     val n = asNumber()
     if (n !is Long) {
@@ -23,6 +27,10 @@ fun LongLiteralExpr.checkedAsLong(): Long {
     return n
 }
 
+fun Long.toHexLiteralExpr(): LongLiteralExpr {
+    return LongLiteralExpr("0x${java.lang.Long.toUnsignedString(this, 16).toUpperCase()}L")
+}
+
 fun Expression.isIntegerOrLongLiteral(): Boolean {
     return isIntegerLiteralExpr || isLongLiteralExpr
 }