Convert integers to char literals

Some character literals are treated as integers by Fernflower, as the
complement operator always evaluates to an integer, even if its operand
is a character. This is fine, but ComplementTransformer removes some
complement transformers - leaving integer literals where Fernflower
would normally insert a character literal.

This transformer converts complemented integers to char literals in some
cases where it makes sense (it is difficult to perform this
transformation across every type of expression in JavaParser).

Signed-off-by: Graham <gpe@openrs2.dev>

deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscatorModule.kt

@@ -8,6 +8,7 @@ import dev.openrs2.deob.ast.gl.GlRegistryProvider
 import dev.openrs2.deob.ast.transform.AddSubTransformer
 import dev.openrs2.deob.ast.transform.BinaryExprOrderTransformer
 import dev.openrs2.deob.ast.transform.BitMaskTransformer
+import dev.openrs2.deob.ast.transform.CharLiteralTransformer
 import dev.openrs2.deob.ast.transform.ComplementTransformer
 import dev.openrs2.deob.ast.transform.EncloseTransformer
 import dev.openrs2.deob.ast.transform.ForLoopConditionTransformer
@@ -33,6 +34,7 @@ object AstDeobfuscatorModule : AbstractModule() {
         binder.addBinding().to(UnencloseTransformer::class.java)
         binder.addBinding().to(NegativeLiteralTransformer::class.java)
         binder.addBinding().to(ComplementTransformer::class.java)
+        binder.addBinding().to(CharLiteralTransformer::class.java)
         binder.addBinding().to(IfElseTransformer::class.java)
         binder.addBinding().to(TernaryTransformer::class.java)
         binder.addBinding().to(BinaryExprOrderTransformer::class.java)

deob-ast/src/main/java/dev/openrs2/deob/ast/transform/CharLiteralTransformer.kt

@@ -0,0 +1,86 @@
+package dev.openrs2.deob.ast.transform
+
+import com.github.javaparser.ast.CompilationUnit
+import com.github.javaparser.ast.expr.AssignExpr
+import com.github.javaparser.ast.expr.BinaryExpr
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.EQUALS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.GREATER
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.GREATER_EQUALS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.LESS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.LESS_EQUALS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.MINUS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.NOT_EQUALS
+import com.github.javaparser.ast.expr.BinaryExpr.Operator.PLUS
+import com.github.javaparser.ast.expr.CharLiteralExpr
+import com.github.javaparser.ast.expr.Expression
+import com.github.javaparser.ast.expr.IntegerLiteralExpr
+import com.github.javaparser.resolution.types.ResolvedPrimitiveType
+import dev.openrs2.deob.ast.Library
+import dev.openrs2.deob.ast.LibraryGroup
+import dev.openrs2.deob.ast.util.walk
+import java.lang.Character.CONTROL
+import java.lang.Character.FORMAT
+import java.lang.Character.LINE_SEPARATOR
+import java.lang.Character.PARAGRAPH_SEPARATOR
+import java.lang.Character.PRIVATE_USE
+import java.lang.Character.SURROGATE
+import java.lang.Character.UNASSIGNED
+
+class CharLiteralTransformer : Transformer() {
+    override fun transformUnit(group: LibraryGroup, library: Library, unit: CompilationUnit) {
+        unit.walk { expr: BinaryExpr ->
+            if (expr.operator in COMPARISON_OPERATORS || expr.operator == PLUS || expr.operator == MINUS) {
+                convertToCharLiteral(expr.left, expr.right)
+                convertToCharLiteral(expr.right, expr.left)
+            }
+        }
+
+        unit.walk { expr: AssignExpr ->
+            convertToCharLiteral(expr.target, expr.value)
+        }
+    }
+
+    private fun convertToCharLiteral(a: Expression, b: Expression) {
+        if (a.calculateResolvedType() != ResolvedPrimitiveType.CHAR) {
+            return
+        } else if (b !is IntegerLiteralExpr) {
+            return
+        }
+
+        b.replace(CharLiteralExpr(escape(b.asNumber().toChar())))
+    }
+
+    private fun escape(c: Char): String {
+        // compatible with Fernflower's character escape code
+        return when (c) {
+            '\b' -> "\\b"
+            '\t' -> "\\t"
+            '\n' -> "\\n"
+            '\u000c' -> "\\f"
+            'r' -> "\\r"
+            '\'' -> "'"
+            '\\' -> "\\"
+            else -> {
+                val type = Character.getType(c).toByte()
+                if (type in UNPRINTABLE_TYPES) {
+                    "\\u" + Integer.toHexString(c.toInt()).padStart(4, '0')
+                } else {
+                    c.toString()
+                }
+            }
+        }
+    }
+
+    private companion object {
+        private val COMPARISON_OPERATORS = setOf(EQUALS, NOT_EQUALS, LESS, GREATER, LESS_EQUALS, GREATER_EQUALS)
+        private val UNPRINTABLE_TYPES = setOf(
+            UNASSIGNED,
+            LINE_SEPARATOR,
+            PARAGRAPH_SEPARATOR,
+            CONTROL,
+            FORMAT,
+            PRIVATE_USE,
+            SURROGATE
+        )
+    }
+}