Convert Java 1.1-style synchronized blocks to a more modern style

The non-HD client appears to have been compiled with javac 1.1 or 1.2,
which compiles synchronized blocks differently to modern versions of
Java. For example, it uses a single try/catch block (modern versions use
two, the latter of which is recursive), subroutines (modern versions do
not) and the range of the try block is narrower.

Fernflower doesn't understand this old style of bytecode, and produces
empty synchronized blocks that do not cover the correct range.

The MONITORENTER sequence is also slightly different: it uses
ASTORE ALOAD MONITORENTER after pushing the monitor variable, rather
than DUP ASTORE MONITORENTER. This doesn't break Fernflower, but does
make it introduce a pointless assignment to a variable only used in the
synchronized() statement, rather than inlining the expression.

This commit introduces a transformer which fixes up Java 1.1-style
synchronized blocks just enough for Fernflower to be able to decompile
them cleanly. (It doesn't add a recursive try/catch, as Fernflower
ignores them.)

Signed-off-by: Graham <gpe@openrs2.dev>

deob/src/main/java/dev/openrs2/deob/Deobfuscator.kt

@@ -23,6 +23,7 @@ import dev.openrs2.deob.transform.FieldOrderTransformer
 import dev.openrs2.deob.transform.FinalTransformer
 import dev.openrs2.deob.transform.InvokeSpecialTransformer
 import dev.openrs2.deob.transform.MethodOrderTransformer
+import dev.openrs2.deob.transform.MonitorTransformer
 import dev.openrs2.deob.transform.OpaquePredicateTransformer
 import dev.openrs2.deob.transform.OriginalNameTransformer
 import dev.openrs2.deob.transform.OriginalPcRestoreTransformer
@@ -154,6 +155,7 @@ class Deobfuscator(private val input: Path, private val output: Path) {
             ResourceTransformer(),
             OpaquePredicateTransformer(),
             ExceptionTracingTransformer(),
+            MonitorTransformer(),
             BitShiftTransformer(),
             CanvasTransformer(),
             FieldOrderTransformer(),

deob/src/main/java/dev/openrs2/deob/transform/MonitorTransformer.kt

@@ -0,0 +1,115 @@
+package dev.openrs2.deob.transform
+
+import com.github.michaelbull.logging.InlineLogger
+import dev.openrs2.asm.InsnMatcher
+import dev.openrs2.asm.classpath.ClassPath
+import dev.openrs2.asm.classpath.Library
+import dev.openrs2.asm.nextReal
+import dev.openrs2.asm.previousReal
+import dev.openrs2.asm.transform.Transformer
+import org.objectweb.asm.Opcodes
+import org.objectweb.asm.tree.AbstractInsnNode
+import org.objectweb.asm.tree.ClassNode
+import org.objectweb.asm.tree.InsnNode
+import org.objectweb.asm.tree.JumpInsnNode
+import org.objectweb.asm.tree.LabelNode
+import org.objectweb.asm.tree.MethodNode
+
+class MonitorTransformer : Transformer() {
+    private var subroutinesInlined = 0
+    private var tryRangesExtended = 0
+    private var loadsReplaced = 0
+
+    override fun preTransform(classPath: ClassPath) {
+        subroutinesInlined = 0
+        tryRangesExtended = 0
+        loadsReplaced = 0
+    }
+
+    override fun transformCode(classPath: ClassPath, library: Library, clazz: ClassNode, method: MethodNode): Boolean {
+        inlineSubroutines(method)
+        extendTryRanges(method)
+        replaceLoadWithDup(method)
+        return false
+    }
+
+    private fun inlineSubroutines(method: MethodNode) {
+        val subroutines = mutableMapOf<AbstractInsnNode, List<AbstractInsnNode>>()
+        for (match in SUBROUTINE_MATCHER.match(method.instructions)) {
+            subroutines[match[0]] = match
+        }
+
+        for (match in JSR_MATCHER.match(method.instructions)) {
+            val jsr = match[1] as JumpInsnNode
+            val subroutine = subroutines[jsr.label.nextReal] ?: continue
+
+            val ret = subroutine[3] as JumpInsnNode
+            if (ret.label.nextReal != jsr.nextReal) {
+                continue
+            }
+
+            val clonedLabels = emptyMap<LabelNode, LabelNode>()
+            method.instructions.set(match[0], subroutine[1].clone(clonedLabels))
+            method.instructions.set(match[1], subroutine[2].clone(clonedLabels))
+
+            subroutinesInlined++
+        }
+    }
+
+    private fun extendTryRanges(method: MethodNode) {
+        for (tryCatch in method.tryCatchBlocks) {
+            if (tryCatch.type != null) {
+                continue
+            }
+
+            val monitorenter = tryCatch.start.previousReal ?: continue
+            if (monitorenter.opcode != Opcodes.MONITORENTER) {
+                continue
+            }
+
+            // extend the try to cover the ALOAD and MONTITOREXIT instructions
+            val aload = tryCatch.end.nextReal ?: continue
+            if (aload.opcode != Opcodes.ALOAD) {
+                continue
+            }
+
+            val monitorexit = aload.nextReal ?: continue
+            if (monitorexit.opcode != Opcodes.MONITOREXIT) {
+                continue
+            }
+
+            val end = monitorexit.nextReal ?: continue
+
+            val label = LabelNode()
+            method.instructions.insertBefore(end, label)
+            tryCatch.end = label
+
+            tryRangesExtended++
+        }
+    }
+
+    private fun replaceLoadWithDup(method: MethodNode) {
+        for (match in LOAD_MATCHER.match(method.instructions)) {
+            method.instructions.insertBefore(match[0], InsnNode(Opcodes.DUP))
+            method.instructions.remove(match[1])
+
+            loadsReplaced++
+        }
+    }
+
+    override fun postTransform(classPath: ClassPath) {
+        logger.info { "Inlined $subroutinesInlined MONITOREXIT subroutines" }
+        logger.info { "Extended $tryRangesExtended try ranges to cover MONITOREXIT instructions" }
+        logger.info { "Replaced $loadsReplaced ASTORE ALOAD sequences with DUP ASTORE" }
+    }
+
+    companion object {
+        private val logger = InlineLogger()
+
+        // these regexes rely on JSRInlinerAdapter running first
+        private val JSR_MATCHER = InsnMatcher.compile("ACONST_NULL GOTO")
+        private val SUBROUTINE_MATCHER = InsnMatcher.compile("ASTORE ALOAD MONITOREXIT GOTO")
+
+        private val LOAD_MATCHER = InsnMatcher.compile("ASTORE ALOAD MONITORENTER")
+    }
+}