Decompile bytecode into basic blocks

Adds support for decompiling bytecode into basic blocks based on the
control flow graph of the code. Also adds a new deob-ir create, intended
as an easier to work with intermediate representation of program code.

Signed-off-by: Gary Tierney <gary.tierney@fastmail.com>

deob-ir/build.gradle.kts

@@ -0,0 +1,28 @@
+plugins {
+    `maven-publish`
+    application
+    kotlin("jvm")
+}
+
+dependencies {
+    implementation(project(":asm"))
+    implementation(project(":deob"))
+    implementation(project(":deob-annotations"))
+    implementation("com.google.guava:guava:${Versions.guava}")
+}
+
+publishing {
+    publications.create<MavenPublication>("maven") {
+        from(components["java"])
+
+        pom {
+            packaging = "jar"
+            name.set("OpenRS2 Deobfuscator IR")
+            description.set(
+                """
+                An intermediate reprsentation of bytecode.
+            """.trimIndent()
+            )
+        }
+    }
+}

deob-ir/src/main/java/dev/openrs2/deob/ir/Method.kt

@@ -0,0 +1,11 @@
+package dev.openrs2.deob.ir
+
+import com.google.common.graph.MutableGraph
+import dev.openrs2.deob.ir.flow.BasicBlock
+import org.objectweb.asm.tree.ClassNode
+import org.objectweb.asm.tree.MethodNode
+
+class Method(val owner: ClassNode, val method: MethodNode, val entry: BasicBlock) {
+    val cfg: MutableGraph<BasicBlock>
+        get() = entry.graph
+}

deob-ir/src/main/java/dev/openrs2/deob/ir/flow/BasicBlock.kt

@@ -0,0 +1,48 @@
+package dev.openrs2.deob.ir.flow
+
+import com.google.common.graph.MutableGraph
+import org.objectweb.asm.tree.InsnList
+import org.objectweb.asm.tree.LabelNode
+import org.objectweb.asm.util.Textifier
+import org.objectweb.asm.util.TraceMethodVisitor
+
+class BasicBlock(
+    internal val graph: MutableGraph<BasicBlock>,
+    private val code: InsnList,
+    private var start: Int,
+    private var end: Int
+) {
+    fun successors() : Set<BasicBlock> = graph.successors(this)
+    fun predecessors() : Set<BasicBlock> = graph.predecessors(this)
+
+    var label: LabelNode? = null
+
+    fun merge(other: BasicBlock) {
+        for (succ in other.successors()) {
+            graph.putEdge(this, succ)
+        }
+
+        graph.removeEdge(this, other)
+        graph.removeNode(other)
+
+        end = other.end
+    }
+
+    override fun equals(other: Any?): Boolean {
+        return this === other
+    }
+
+    override fun hashCode(): Int {
+        return System.identityHashCode(this)
+    }
+
+    override fun toString(): String {
+        val textifier = Textifier()
+        val methodTracer = TraceMethodVisitor(textifier)
+
+        code.accept(methodTracer)
+
+        val text = textifier.text.subList(start, end + 1) as List<String>
+        return text.reduce { l, r -> l + r }
+    }
+}

deob-ir/src/main/java/dev/openrs2/deob/ir/translation/IrDecompiler.kt

@@ -0,0 +1,89 @@
+package dev.openrs2.deob.ir.translation
+
+import com.google.common.graph.GraphBuilder
+import dev.openrs2.deob.analysis.ControlFlowAnalyzer
+import dev.openrs2.deob.ir.Method
+import dev.openrs2.deob.ir.flow.BasicBlock
+import org.objectweb.asm.tree.ClassNode
+import org.objectweb.asm.tree.LabelNode
+import org.objectweb.asm.tree.MethodNode
+import org.objectweb.asm.tree.analysis.Analyzer
+import org.objectweb.asm.tree.analysis.BasicInterpreter
+import org.objectweb.asm.tree.analysis.BasicValue
+import java.util.Stack
+
+class IrDecompiler(private val owner: ClassNode, private val method: MethodNode) :
+    Analyzer<BasicValue>(BasicInterpreter()) {
+
+    private val graph = GraphBuilder
+        .directed()
+        .allowsSelfLoops(true)
+        .build<BasicBlock>()
+
+    private val blocks = mutableMapOf<Int, BasicBlock>()
+    private fun bb(index: Int) = blocks.computeIfAbsent(index) {
+        val bb = BasicBlock(graph, method.instructions, it, it)
+        val leader = method.instructions[index]
+
+        if (leader is LabelNode) {
+            bb.label = leader
+        }
+
+        bb
+    }
+
+    override fun newControlFlowEdge(insnIndex: Int, successorIndex: Int) {
+        graph.putEdge(bb(insnIndex), bb(successorIndex))
+    }
+
+    override fun newControlFlowExceptionEdge(insnIndex: Int, successorIndex: Int): Boolean {
+        // @TODO: Attach basic blocks with exception types.
+        return true
+    }
+
+    fun decompile(): Method {
+        analyze(owner.name, method)
+
+        val entryBlock = blocks[0] ?: throw IllegalStateException("No method entry block found")
+
+        val remainingBlocks = Stack<BasicBlock>()
+        remainingBlocks.push(entryBlock)
+
+        val visited = mutableSetOf<BasicBlock>()
+
+        while (remainingBlocks.isNotEmpty()) {
+            val bb = remainingBlocks.pop()
+            var next = bb.findTrivialSuccessor()
+
+            while (next != null && next != bb) {
+                bb.merge(next)
+
+                next = bb.findTrivialSuccessor()
+            }
+
+            for (succ in bb.successors()) {
+                if (!visited.contains(succ)) {
+                    remainingBlocks.push(succ)
+                }
+            }
+
+            visited.add(bb)
+        }
+
+        return Method(owner, method, entryBlock)
+    }
+}
+
+private fun BasicBlock.findTrivialSuccessor(): BasicBlock? {
+    val successors = successors()
+    if (successors.size != 1) return null
+
+    val successor = successors.first()!!
+    val nextPredecessors = successor.predecessors()
+
+    if (nextPredecessors.size == 1) {
+        return successor
+    } else {
+        return null
+    }
+}

deob-ir/src/test/java/dev/openrs2/deob/ir/translation/IrDecompilerTests.kt

@@ -0,0 +1,27 @@
+package dev.openrs2.deob.ir.translation
+
+import dev.openrs2.deob.ir.translation.fixture.Fixture
+import dev.openrs2.deob.ir.translation.fixture.FixtureMethod
+import org.junit.jupiter.api.Test
+
+class IrDecompilerTests {
+
+    @Test
+    fun `Creates entry basic block`() {
+        class CfgSample(val a: Boolean, val b: Boolean) :
+            Fixture {
+            override fun test() {
+                if (a) {
+                    println("a")
+                } else if (b) {
+                    println("b")
+                }
+            }
+        }
+
+        val fixture = FixtureMethod.from(CfgSample::class)
+        val decompiler = IrDecompiler(fixture.owner, fixture.method)
+        val irMethod = decompiler.decompile()
+    }
+}
+

deob-ir/src/test/java/dev/openrs2/deob/ir/translation/fixture/Fixture.java

@@ -0,0 +1,10 @@
+package dev.openrs2.deob.ir.translation.fixture;
+
+import dev.openrs2.deob.ir.translation.IrDecompilerTests;
+
+/**
+ * Marker interface for {@link IrDecompilerTests}.
+ */
+public interface Fixture {
+	void test();
+}

deob-ir/src/test/java/dev/openrs2/deob/ir/translation/fixture/FixtureMethod.kt

@@ -0,0 +1,28 @@
+package dev.openrs2.deob.ir.translation.fixture
+
+import org.objectweb.asm.ClassReader
+import org.objectweb.asm.tree.ClassNode
+import org.objectweb.asm.tree.MethodNode
+import java.io.File
+import kotlin.reflect.KClass
+
+class FixtureMethod private constructor(val owner: ClassNode, val method: MethodNode) {
+    companion object {
+        fun from(ty: KClass<out Fixture>): FixtureMethod {
+            val classPath = "/${ty.java.name.replace('.', File.separatorChar)}.class"
+            val classFile = ty.java.getResourceAsStream(classPath)
+
+            return classFile.use {
+                val node = ClassNode()
+                val reader = ClassReader(classFile)
+
+                reader.accept(node, ClassReader.SKIP_DEBUG)
+
+                val method = node.methods.find { it.name == "test" } ?:
+                    throw IllegalStateException("Fixture class has no test() method")
+
+                return FixtureMethod(node, method)
+            }
+        }
+    }
+}

settings.gradle.kts

@@ -11,6 +11,7 @@ include(
     "deob",
     "deob-annotations",
     "deob-ast",
+    "deob-ir",
     "game"
 )