From 7d9b1c61e81d26b3242d52add4c1b759fd6a25bc Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 30 May 2020 21:10:37 +0100 Subject: [PATCH] Combine the StaticScramblingTransformer with the RemapTransformer This change paves the way to feed the NameMap into the TypedRemapper, such that refactored names (and static member movements) are preserved across deobfuscator runs. Signed-off-by: Graham --- .../java/dev/openrs2/asm/classpath/Library.kt | 10 +- .../openrs2/asm/classpath/LibraryRemapper.kt | 219 ++++++++++++++ .../dev/openrs2/deob/DeobfuscatorModule.kt | 6 +- .../transform/InvokeSpecialTransformer.kt | 12 +- .../transform/StaticScramblingTransformer.kt | 276 ------------------ 5 files changed, 230 insertions(+), 293 deletions(-) create mode 100644 asm/src/main/java/dev/openrs2/asm/classpath/LibraryRemapper.kt delete mode 100644 deob/src/main/java/dev/openrs2/deob/transform/StaticScramblingTransformer.kt diff --git a/asm/src/main/java/dev/openrs2/asm/classpath/Library.kt b/asm/src/main/java/dev/openrs2/asm/classpath/Library.kt index 8d3232c9..4dd3b4e5 100644 --- a/asm/src/main/java/dev/openrs2/asm/classpath/Library.kt +++ b/asm/src/main/java/dev/openrs2/asm/classpath/Library.kt @@ -3,14 +3,14 @@ package dev.openrs2.asm.classpath import com.github.michaelbull.logging.InlineLogger import dev.openrs2.asm.io.LibraryReader import dev.openrs2.asm.io.LibraryWriter -import dev.openrs2.asm.remap import org.objectweb.asm.tree.ClassNode import java.nio.file.Files import java.nio.file.Path +import java.util.SortedMap import java.util.TreeMap class Library() : Iterable { - private var classes = TreeMap() + private var classes: SortedMap = TreeMap() constructor(library: Library) : this() { for (clazz in library.classes.values) { @@ -41,11 +41,7 @@ class Library() : Iterable { } fun remap(remapper: ExtendedRemapper) { - for (clazz in classes.values) { - clazz.remap(remapper) - } - - classes = classes.mapKeysTo(TreeMap()) { (_, clazz) -> clazz.name } + classes = LibraryRemapper(remapper, classes).remap() } fun write(path: Path, writer: LibraryWriter, classPath: ClassPath) { diff --git a/asm/src/main/java/dev/openrs2/asm/classpath/LibraryRemapper.kt b/asm/src/main/java/dev/openrs2/asm/classpath/LibraryRemapper.kt new file mode 100644 index 00000000..e3bf8284 --- /dev/null +++ b/asm/src/main/java/dev/openrs2/asm/classpath/LibraryRemapper.kt @@ -0,0 +1,219 @@ +package dev.openrs2.asm.classpath + +import dev.openrs2.asm.ClassVersionUtils +import dev.openrs2.asm.MemberRef +import dev.openrs2.asm.remap +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.ClassNode +import org.objectweb.asm.tree.FieldInsnNode +import org.objectweb.asm.tree.FieldNode +import org.objectweb.asm.tree.InsnList +import org.objectweb.asm.tree.InsnNode +import org.objectweb.asm.tree.MethodNode +import java.util.SortedMap +import java.util.TreeMap +import kotlin.math.max + +class LibraryRemapper( + private val remapper: ExtendedRemapper, + private var classes: SortedMap +) { + private class Initializer(val instructions: InsnList, val maxStack: Int) { + val dependencies = instructions.asSequence() + .filterIsInstance() + .filter { it.opcode == Opcodes.GETSTATIC } + .map(::MemberRef) + .toSet() + } + + private class Field(val owner: String, val node: FieldNode, val version: Int, val initializer: Initializer?) + private class Method(val owner: String, val node: MethodNode, val version: Int) + + private val fields = mutableMapOf() + private val splicedFields = mutableSetOf() + private val methods = mutableListOf() + + fun remap(): SortedMap { + // extract static fields/methods that are being moved between classes + extractFields() + extractMethods() + + // map remaining fields/methods + for (clazz in classes.values) { + clazz.remap(remapper) + } + + classes = classes.mapKeysTo(TreeMap()) { (_, clazz) -> clazz.name } + + // splice static fields/methods into their new classes + spliceFields() + spliceMethods() + + // remove empty methods (so EmptyClassTransformer works later) + removeEmptyClinitMethods() + + return classes + } + + private fun extractFields() { + for (clazz in classes.values) { + clazz.fields.removeIf { field -> + // do nothing if the field is not moved between classes + val oldOwner = remapper.mapType(clazz.name) + val newOwner = remapper.mapFieldOwner(clazz.name, field.name, field.desc) + if (oldOwner == newOwner) { + return@removeIf false + } + + /* + * Remove the initializer (if present) from the old owner's + * method. + */ + val initializer = extractInitializer(clazz, field) + + /* + * Map the field (it won't be caught by a ClassNode::remap call + * during the main pass). + */ + field.remap(remapper, clazz.name) + + /* + * Store the field so it can be spliced into its new class + * later. We key on the new owner/name/descriptor, rather than + * the old one, as spliceFields's dependency tracking runs + * after the Initializer::dependencies has been mapped. + */ + val newMember = MemberRef(newOwner, field.name, field.desc) + fields[newMember] = Field(newOwner, field, clazz.version, initializer) + + // remove the field from its old class + return@removeIf true + } + } + } + + private fun extractInitializer(clazz: ClassNode, field: FieldNode): Initializer? { + val clinit = clazz.methods.find { it.name == "" } ?: return null + val initializer = remapper.getFieldInitializer(clazz.name, field.name, field.desc) ?: return null + + val list = InsnList() + for (insn in initializer) { + /* + * Remove initializer from . It is stored alongside the + * FieldNode so it can be spliced into its new class later. + */ + clinit.instructions.remove(insn) + list.add(insn) + + /* + * Remap the initializer (it won't be caught by a ClassNode::remap + * call during the main pass). + */ + insn.remap(remapper) + } + + return Initializer(list, clinit.maxStack) + } + + private fun extractMethods() { + for (clazz in classes.values) { + clazz.methods.removeIf { method -> + // do nothing if the method is not moved between classes + val oldOwner = remapper.mapType(clazz.name) + val newOwner = remapper.mapMethodOwner(clazz.name, method.name, method.desc) + if (oldOwner == newOwner) { + return@removeIf false + } + + /* + * Map the method (it won't be caught by a ClassNode::remap call + * during the main pass). + */ + method.remap(remapper, clazz.name) + + // store the method + methods += Method(newOwner, method, clazz.version) + + // remove the method from its old class + return@removeIf true + } + } + } + + private fun spliceFields() { + for (member in fields.keys) { + spliceField(member) + } + } + + private fun spliceField(member: MemberRef) { + if (!splicedFields.add(member)) { + return + } + + val field = fields[member] ?: return + val clazz = classes.computeIfAbsent(field.owner, ::createClass) + + if (field.initializer != null) { + for (dependency in field.initializer.dependencies) { + spliceField(dependency) + } + + val clinit = clazz.methods.find { it.name == "" } ?: createClinitMethod(clazz) + // TODO(gpe): check the method only has a single RETURN and that it is the last instruction + clinit.maxStack = max(clinit.maxStack, field.initializer.maxStack) + clinit.instructions.insertBefore(clinit.instructions.last, field.initializer.instructions) + } + + clazz.version = ClassVersionUtils.max(clazz.version, field.version) + clazz.fields.add(field.node) + } + + private fun spliceMethods() { + for (method in methods) { + val clazz = classes.computeIfAbsent(method.owner, ::createClass) + clazz.version = ClassVersionUtils.max(clazz.version, method.version) + clazz.methods.add(method.node) + } + } + + private fun removeEmptyClinitMethods() { + for (clazz in classes.values) { + val clinit = clazz.methods.find { it.name == "" } ?: continue + + val first = clinit.instructions.firstOrNull { it.opcode != -1 } + if (first != null && first.opcode == Opcodes.RETURN) { + clazz.methods.remove(clinit) + } + } + } + + private fun createClass(name: String): ClassNode { + val clazz = ClassNode() + clazz.version = Opcodes.V1_1 + clazz.access = Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER or Opcodes.ACC_FINAL + clazz.name = name + clazz.superName = "java/lang/Object" + clazz.interfaces = mutableListOf() + clazz.innerClasses = mutableListOf() + clazz.fields = mutableListOf() + clazz.methods = mutableListOf() + return clazz + } + + private fun createClinitMethod(clazz: ClassNode): MethodNode { + val clinit = MethodNode() + clinit.access = Opcodes.ACC_STATIC + clinit.name = "" + clinit.desc = "()V" + clinit.exceptions = mutableListOf() + clinit.parameters = mutableListOf() + clinit.instructions = InsnList() + clinit.instructions.add(InsnNode(Opcodes.RETURN)) + clinit.tryCatchBlocks = mutableListOf() + + clazz.methods.add(clinit) + + return clinit + } +} diff --git a/deob/src/main/java/dev/openrs2/deob/DeobfuscatorModule.kt b/deob/src/main/java/dev/openrs2/deob/DeobfuscatorModule.kt index 9636c727..cbde7517 100644 --- a/deob/src/main/java/dev/openrs2/deob/DeobfuscatorModule.kt +++ b/deob/src/main/java/dev/openrs2/deob/DeobfuscatorModule.kt @@ -30,7 +30,6 @@ import dev.openrs2.deob.transform.OverrideTransformer import dev.openrs2.deob.transform.RedundantGotoTransformer import dev.openrs2.deob.transform.RemapTransformer import dev.openrs2.deob.transform.ResetTransformer -import dev.openrs2.deob.transform.StaticScramblingTransformer import dev.openrs2.deob.transform.UnusedArgTransformer import dev.openrs2.deob.transform.UnusedLocalTransformer import dev.openrs2.deob.transform.UnusedMethodTransformer @@ -57,6 +56,8 @@ object DeobfuscatorModule : AbstractModule() { binder.addBinding().to(CanvasTransformer::class.java) binder.addBinding().to(FieldOrderTransformer::class.java) binder.addBinding().to(BitwiseOpTransformer::class.java) + binder.addBinding().to(ClassLiteralTransformer::class.java) + binder.addBinding().to(InvokeSpecialTransformer::class.java) binder.addBinding().to(RemapTransformer::class.java) binder.addBinding().to(ConstantArgTransformer::class.java) binder.addBinding().to(UnusedLocalTransformer::class.java) @@ -64,9 +65,6 @@ object DeobfuscatorModule : AbstractModule() { binder.addBinding().to(UnusedArgTransformer::class.java) binder.addBinding().to(CounterTransformer::class.java) binder.addBinding().to(ResetTransformer::class.java) - binder.addBinding().to(ClassLiteralTransformer::class.java) - binder.addBinding().to(InvokeSpecialTransformer::class.java) - binder.addBinding().to(StaticScramblingTransformer::class.java) binder.addBinding().to(EmptyClassTransformer::class.java) binder.addBinding().to(MethodOrderTransformer::class.java) binder.addBinding().to(VisibilityTransformer::class.java) diff --git a/deob/src/main/java/dev/openrs2/deob/transform/InvokeSpecialTransformer.kt b/deob/src/main/java/dev/openrs2/deob/transform/InvokeSpecialTransformer.kt index e6105d49..7a020eed 100644 --- a/deob/src/main/java/dev/openrs2/deob/transform/InvokeSpecialTransformer.kt +++ b/deob/src/main/java/dev/openrs2/deob/transform/InvokeSpecialTransformer.kt @@ -33,10 +33,10 @@ import javax.inject.Singleton * instruction, and non-virtual function calls are cheaper than virtual * function calls) or an obfuscation technique. * - * The [StaticScramblingTransformer] moves static methods containing these - * `INVOKESPECIAL` instructions to new classes. This change is still permitted - * by the JVM specification, which does not place any restrictions on the class - * referenced by the `INVOKESPECIAL` instruction. + * The [RemapTransformer] moves static methods containing these `INVOKESPECIAL` + * instructions to new classes. This change is still permitted by the JVM + * specification, which does not place any restrictions on the class referenced + * by the `INVOKESPECIAL` instruction. * * However, the * [verifier](https://github.com/openjdk/jdk11u/blob/3789983e89c9de252ef546a1b98a732a7d066650/src/java.base/share/native/libverify/check_code.c#L1342) @@ -45,8 +45,8 @@ import javax.inject.Singleton * direct superclass illegal. * * This transformer replaces `INVOKESPECIAL` with equivalent `INVOKEVIRTUAL` - * instructions where possible, allowing the [StaticScramblingTransformer] to - * produce verifiable output. + * instructions where possible, allowing the [RemapTransformer] to produce + * verifiable output. */ @Singleton class InvokeSpecialTransformer : Transformer() { diff --git a/deob/src/main/java/dev/openrs2/deob/transform/StaticScramblingTransformer.kt b/deob/src/main/java/dev/openrs2/deob/transform/StaticScramblingTransformer.kt deleted file mode 100644 index cc7285c7..00000000 --- a/deob/src/main/java/dev/openrs2/deob/transform/StaticScramblingTransformer.kt +++ /dev/null @@ -1,276 +0,0 @@ -package dev.openrs2.deob.transform - -import com.github.michaelbull.logging.InlineLogger -import dev.openrs2.asm.ClassVersionUtils -import dev.openrs2.asm.MemberDesc -import dev.openrs2.asm.MemberRef -import dev.openrs2.asm.classpath.ClassPath -import dev.openrs2.asm.classpath.Library -import dev.openrs2.asm.getExpression -import dev.openrs2.asm.isSequential -import dev.openrs2.asm.transform.Transformer -import dev.openrs2.deob.Profile -import dev.openrs2.util.collect.DisjointSet -import org.objectweb.asm.Opcodes -import org.objectweb.asm.tree.AbstractInsnNode -import org.objectweb.asm.tree.ClassNode -import org.objectweb.asm.tree.FieldInsnNode -import org.objectweb.asm.tree.FieldNode -import org.objectweb.asm.tree.InsnList -import org.objectweb.asm.tree.InsnNode -import org.objectweb.asm.tree.MethodInsnNode -import org.objectweb.asm.tree.MethodNode -import javax.inject.Inject -import javax.inject.Singleton -import kotlin.math.max - -@Singleton -class StaticScramblingTransformer @Inject constructor(private val profile: Profile) : Transformer() { - private data class Field(val node: FieldNode, val initializer: InsnList, val version: Int, val maxStack: Int) { - val dependencies = initializer.asSequence() - .filterIsInstance() - .filter { it.opcode == Opcodes.GETSTATIC } - .map(::MemberRef) - .toSet() - } - - private lateinit var inheritedFieldSets: DisjointSet - private lateinit var inheritedMethodSets: DisjointSet - private val fields = mutableMapOf, Field>() - private val fieldClasses = mutableMapOf, String>() - private val methodClasses = mutableMapOf, String>() - private var nextStaticClass: ClassNode? = null - private var nextClinit: MethodNode? = null - private val staticClasses = mutableListOf() - - private fun nextClass(): Pair { - var clazz = nextStaticClass - if (clazz != null && clazz.fields.size < MAX_FIELDS && clazz.methods.size < MAX_METHODS) { - return Pair(clazz, nextClinit!!) - } - - val clinit = MethodNode() - clinit.access = Opcodes.ACC_STATIC - clinit.name = "" - clinit.desc = "()V" - clinit.exceptions = mutableListOf() - clinit.parameters = mutableListOf() - clinit.instructions = InsnList() - clinit.instructions.add(InsnNode(Opcodes.RETURN)) - clinit.tryCatchBlocks = mutableListOf() - - clazz = ClassNode() - clazz.version = Opcodes.V1_1 - clazz.access = Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER or Opcodes.ACC_FINAL - clazz.name = "Static${staticClasses.size + 1}" - clazz.superName = "java/lang/Object" - clazz.interfaces = mutableListOf() - clazz.innerClasses = mutableListOf() - clazz.fields = mutableListOf() - clazz.methods = mutableListOf(clinit) - - staticClasses += clazz - nextStaticClass = clazz - nextClinit = clinit - - return Pair(clazz, clinit) - } - - private fun MethodNode.extractEntryExitBlocks(): List { - /* - * Most (or all?) of the methods have "simple" initializers - * that we're capable of moving in the first and last basic blocks of - * the method. The last basic block is always at the end of the code - * and ends in a RETURN. This allows us to avoid worrying about making - * a full basic block control flow graph here. - */ - val entry = instructions.takeWhile { it.isSequential } - - val last = instructions.lastOrNull() - if (last == null || last.opcode != Opcodes.RETURN) { - return entry - } - - val exit = instructions.toList() - .dropLast(1) - .takeLastWhile { it.isSequential } - - return entry.plus(exit) - } - - private fun MethodNode.extractInitializers(owner: String): Pair, Set> { - val entryExitBlocks = extractEntryExitBlocks() - - val simpleInitializers = mutableMapOf() - val complexInitializers = instructions.asSequence() - .filter { !entryExitBlocks.contains(it) } - .filterIsInstance() - .filter { it.opcode == Opcodes.GETSTATIC && it.owner == owner } - .filter { !profile.excludedFields.matches(it.owner, it.name, it.desc) } - .map(::MemberDesc) - .toSet() - - val putstatics = entryExitBlocks - .filterIsInstance() - .filter { it.opcode == Opcodes.PUTSTATIC && it.owner == owner } - .filter { !profile.excludedFields.matches(it.owner, it.name, it.desc) } - - for (putstatic in putstatics) { - val desc = MemberDesc(putstatic) - if (simpleInitializers.containsKey(desc) || complexInitializers.contains(desc)) { - continue - } - - // TODO(gpe): use a filter here (pure with no *LOADs?) - val expr = getExpression(putstatic) ?: continue - - val initializer = InsnList() - for (insn in expr) { - instructions.remove(insn) - initializer.add(insn) - } - instructions.remove(putstatic) - initializer.add(putstatic) - - simpleInitializers[desc] = initializer - } - - return Pair(simpleInitializers, complexInitializers) - } - - private fun spliceInitializers() { - val done = mutableSetOf>() - for ((partition, field) in fields) { - spliceInitializers(done, partition, field) - } - } - - private fun spliceInitializers( - done: MutableSet>, - partition: DisjointSet.Partition, - field: Field - ) { - if (!done.add(partition)) { - return - } - - for (dependency in field.dependencies) { - val dependencyPartition = inheritedFieldSets[dependency]!! - val dependencyField = fields[dependencyPartition] ?: continue - spliceInitializers(done, partition, dependencyField) - } - - val (staticClass, clinit) = nextClass() - staticClass.fields.add(field.node) - staticClass.version = ClassVersionUtils.max(staticClass.version, field.version) - clinit.instructions.insertBefore(clinit.instructions.last, field.initializer) - clinit.maxStack = max(clinit.maxStack, field.maxStack) - - fieldClasses[partition] = staticClass.name - } - - override fun preTransform(classPath: ClassPath) { - inheritedFieldSets = classPath.createInheritedFieldSets() - inheritedMethodSets = classPath.createInheritedMethodSets() - fields.clear() - fieldClasses.clear() - methodClasses.clear() - nextStaticClass = null - staticClasses.clear() - - for (library in classPath.libraries) { - // TODO(gpe): improve detection of the client library - if ("client" !in library) { - continue - } - - for (clazz in library) { - // TODO(gpe): exclude the JSObject class - - val clinit = clazz.methods.find { it.name == "" } - val (simpleInitializers, complexInitializers) = clinit?.extractInitializers(clazz.name) - ?: Pair(emptyMap(), emptySet()) - - clazz.fields.removeIf { field -> - if (field.access and Opcodes.ACC_STATIC == 0) { - return@removeIf false - } else if (profile.excludedFields.matches(clazz.name, field.name, field.desc)) { - return@removeIf false - } - - val desc = MemberDesc(field) - if (complexInitializers.contains(desc)) { - return@removeIf false - } - - val initializer = simpleInitializers[desc] ?: InsnList() - val maxStack = clinit?.maxStack ?: 0 - - val partition = inheritedFieldSets[MemberRef(clazz, field)]!! - fields[partition] = Field(field, initializer, clazz.version, maxStack) - return@removeIf true - } - - clazz.methods.removeIf { method -> - if (method.access and Opcodes.ACC_STATIC == 0) { - return@removeIf false - } else if (method.access and Opcodes.ACC_NATIVE != 0) { - return@removeIf false - } else if (profile.excludedMethods.matches(clazz.name, method.name, method.desc)) { - return@removeIf false - } - - val (staticClass, _) = nextClass() - staticClass.methods.add(method) - staticClass.version = ClassVersionUtils.max(staticClass.version, clazz.version) - - val partition = inheritedMethodSets[MemberRef(clazz, method)]!! - methodClasses[partition] = staticClass.name - return@removeIf true - } - - val first = clinit?.instructions?.firstOrNull { it.opcode != -1 } - if (first != null && first.opcode == Opcodes.RETURN) { - clazz.methods.remove(clinit) - } - } - - spliceInitializers() - - for (clazz in staticClasses) { - library.add(clazz) - } - } - } - - override fun transformCode(classPath: ClassPath, library: Library, clazz: ClassNode, method: MethodNode): Boolean { - for (insn in method.instructions) { - when (insn) { - is FieldInsnNode -> { - val partition = inheritedFieldSets[MemberRef(insn)] - if (partition != null) { - insn.owner = fieldClasses.getOrDefault(partition, insn.owner) - } - } - is MethodInsnNode -> { - val partition = inheritedMethodSets[MemberRef(insn)] - if (partition != null) { - insn.owner = methodClasses.getOrDefault(partition, insn.owner) - } - } - } - } - - return false - } - - override fun postTransform(classPath: ClassPath) { - logger.info { "Moved ${fieldClasses.size} fields and ${methodClasses.size} methods" } - } - - private companion object { - private val logger = InlineLogger() - private const val MAX_FIELDS = 500 - private const val MAX_METHODS = 50 - } -}