Remove mutually-recursive dummy method calls

This is a little bit grim and probably not completely safe in all cases,
but it works well enough on the client.

Ideally I think I'd do it with a dominator tree calculated from a call
graph aware of integer constants and conditional calls, but that's quite
complicated (especially given how the existing code in the
DummyArgTransformer works).

deob/src/main/java/dev/openrs2/deob/transform/DummyArgTransformer.java

@@ -6,10 +6,12 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
+import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
+import dev.openrs2.asm.InsnMatcher;
 import dev.openrs2.asm.InsnNodeUtils;
 import dev.openrs2.asm.MemberRef;
 import dev.openrs2.asm.StackMetadata;
@@ -27,6 +29,7 @@ import org.objectweb.asm.tree.ClassNode;
 import org.objectweb.asm.tree.JumpInsnNode;
 import org.objectweb.asm.tree.MethodInsnNode;
 import org.objectweb.asm.tree.MethodNode;
+import org.objectweb.asm.tree.VarInsnNode;
 import org.objectweb.asm.tree.analysis.Analyzer;
 import org.objectweb.asm.tree.analysis.AnalyzerException;
 import org.slf4j.Logger;
@@ -35,6 +38,34 @@ import org.slf4j.LoggerFactory;
 public final class DummyArgTransformer extends Transformer {
 	private static final Logger logger = LoggerFactory.getLogger(DummyArgTransformer.class);
 
+	private static final InsnMatcher CONDITIONAL_CALL_MATCHER = InsnMatcher.compile("ILOAD (IFEQ | IFNE | (ICONST | BIPUSH | SIPUSH | LDC) (IF_ICMPEQ | IF_ICMPNE | IF_ICMPLT | IF_ICMPGE | IF_ICMPGT | IF_ICMPLE)) ALOAD? (ICONST | FCONST | DCONST | BIPUSH | SIPUSH | LDC | ACONST_NULL CHECKCAST)+ (INVOKEVIRTUAL | INVOKESTATIC | INVOKEINTERFACE)");
+
+	private static final class ConditionalCall {
+		private final int conditionVar, conditionOpcode;
+		private final Integer conditionValue;
+		private final DisjointSet.Partition<MemberRef> method;
+		private final Integer[] constArgs;
+
+		public ConditionalCall(int conditionVar, int conditionOpcode, Integer conditionValue, DisjointSet.Partition<MemberRef> method, Integer[] constArgs) {
+			this.conditionVar = conditionVar;
+			this.conditionOpcode = conditionOpcode;
+			this.conditionValue = conditionValue;
+			this.method = method;
+			this.constArgs = constArgs;
+		}
+
+		@Override
+		public String toString() {
+			return MoreObjects.toStringHelper(this)
+				.add("conditionVar", conditionVar)
+				.add("conditionOpcode", conditionOpcode)
+				.add("conditionValue", conditionValue)
+				.add("method", method)
+				.add("constArgs", constArgs)
+				.toString();
+		}
+	}
+
 	private enum BranchResult {
 		ALWAYS_TAKEN,
 		NEVER_TAKEN,
@@ -115,7 +146,56 @@ public final class DummyArgTransformer extends Transformer {
 		}
 	}
 
-	private static ImmutableSet<Integer> union(DisjointSet.Partition<MemberRef> method, Collection<SourcedIntValue> intValues) {
+	private final Multimap<ArgRef, SourcedIntValue> argValues = HashMultimap.create();
+	private final Multimap<DisjointSet.Partition<MemberRef>, ConditionalCall> conditionalCalls = HashMultimap.create();
+	private final Map<DisjointSet.Partition<MemberRef>, ImmutableSet<Integer>[]> constArgs = new HashMap<>();
+	private DisjointSet<MemberRef> inheritedMethodSets;
+	private int branchesSimplified, constantsInlined;
+
+	private boolean isMutuallyRecursiveDummy(DisjointSet.Partition<MemberRef> method, int arg, DisjointSet.Partition<MemberRef> source, int value) {
+		for (var sourceToMethodCall : conditionalCalls.get(source)) {
+			if (!sourceToMethodCall.method.equals(method)) {
+				continue;
+			}
+
+			for (var methodToSourceCall : conditionalCalls.get(method)) {
+				if (!methodToSourceCall.method.equals(source)) {
+					continue;
+				}
+
+				if (methodToSourceCall.conditionVar != arg) {
+					continue;
+				}
+
+				boolean taken;
+				if (methodToSourceCall.conditionValue != null) {
+					taken = evaluateBinaryBranch(methodToSourceCall.conditionOpcode, value, methodToSourceCall.conditionValue);
+				} else {
+					taken = evaluateUnaryBranch(methodToSourceCall.conditionOpcode, value);
+				}
+
+				if (taken) {
+					continue;
+				}
+
+				if (sourceToMethodCall.conditionValue != null) {
+					taken = evaluateBinaryBranch(sourceToMethodCall.conditionOpcode, methodToSourceCall.constArgs[sourceToMethodCall.conditionVar], sourceToMethodCall.conditionValue);
+				} else {
+					taken = evaluateUnaryBranch(sourceToMethodCall.conditionOpcode, methodToSourceCall.constArgs[sourceToMethodCall.conditionVar]);
+				}
+
+				if (taken) {
+					continue;
+				}
+
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	private ImmutableSet<Integer> union(DisjointSet.Partition<MemberRef> method, int arg, Collection<SourcedIntValue> intValues) {
 		var builder = ImmutableSet.<Integer>builder();
 
 		for (var value : intValues) {
@@ -129,6 +209,12 @@ public final class DummyArgTransformer extends Transformer {
 				continue;
 			}
 
+			if (intValue.isSingleConstant()) {
+				if (isMutuallyRecursiveDummy(method, arg, source, intValue.getIntValue())) {
+					continue;
+				}
+			}
+
 			builder.addAll(intValue.getIntValues());
 		}
 
@@ -140,11 +226,6 @@ public final class DummyArgTransformer extends Transformer {
 		return set;
 	}
 
-	private final Multimap<ArgRef, SourcedIntValue> argValues = HashMultimap.create();
-	private final Map<DisjointSet.Partition<MemberRef>, ImmutableSet<Integer>[]> constArgs = new HashMap<>();
-	private DisjointSet<MemberRef> inheritedMethodSets;
-	private int branchesSimplified, constantsInlined;
-
 	@Override
 	protected void preTransform(ClassPath classPath) {
 		inheritedMethodSets = classPath.createInheritedMethodSets();
@@ -155,11 +236,78 @@ public final class DummyArgTransformer extends Transformer {
 	@Override
 	protected void prePass(ClassPath classPath) {
 		argValues.clear();
+		conditionalCalls.clear();
 	}
 
 	@Override
 	protected boolean transformCode(ClassPath classPath, Library library, ClassNode clazz, MethodNode method) throws AnalyzerException {
 		var parentMethod = inheritedMethodSets.get(new MemberRef(clazz.name, method.name, method.desc));
+
+		var stores = new boolean[method.maxLocals];
+
+		for (var it = method.instructions.iterator(); it.hasNext(); ) {
+			var insn = it.next();
+
+			var opcode = insn.getOpcode();
+			if (opcode != Opcodes.ISTORE) {
+				continue;
+			}
+
+			var store = (VarInsnNode) insn;
+			stores[store.var] = true;
+		}
+
+		CONDITIONAL_CALL_MATCHER.match(method).forEach(match -> {
+			var matchIndex = 0;
+			var load = (VarInsnNode) match.get(matchIndex++);
+			if (stores[load.var]) {
+				return;
+			}
+
+			var callerSlots = Type.getArgumentsAndReturnSizes(method.desc) >> 2;
+			if ((method.access & Opcodes.ACC_STATIC) != 0) {
+				callerSlots++;
+			}
+
+			if (load.var >= callerSlots) {
+				return;
+			}
+
+			Integer conditionValue;
+			var conditionOpcode = match.get(matchIndex).getOpcode();
+			if (conditionOpcode == Opcodes.IFEQ || conditionOpcode == Opcodes.IFNE) {
+				conditionValue = null;
+				matchIndex++;
+			} else {
+				conditionValue = InsnNodeUtils.getIntConstant(match.get(matchIndex++));
+				conditionOpcode = match.get(matchIndex++).getOpcode();
+			}
+
+			var invoke = (MethodInsnNode) match.get(match.size() - 1);
+
+			var invokeArgTypes = Type.getArgumentTypes(invoke.desc).length;
+			var constArgs = new Integer[invokeArgTypes];
+
+			if (invoke.getOpcode() != Opcodes.INVOKESTATIC) {
+				matchIndex++;
+			}
+
+			for (int i = 0; i < constArgs.length; i++) {
+				var insn = match.get(matchIndex++);
+				if (insn.getOpcode() == Opcodes.ACONST_NULL) {
+					matchIndex++;
+				} else if (InsnNodeUtils.isIntConstant(insn)) {
+					constArgs[i] = InsnNodeUtils.getIntConstant(insn);
+				}
+			}
+
+			var callee = inheritedMethodSets.get(new MemberRef(invoke.owner, invoke.name, invoke.desc));
+			if (callee == null) {
+				return;
+			}
+			conditionalCalls.put(parentMethod, new ConditionalCall(load.var, conditionOpcode, conditionValue, callee, constArgs));
+		});
+
 		var parameters = constArgs.get(parentMethod);
 
 		var analyzer = new Analyzer<>(new IntInterpreter(parameters));
@@ -303,7 +451,7 @@ public final class DummyArgTransformer extends Transformer {
 			@SuppressWarnings("unchecked")
 			var parameters = (ImmutableSet<Integer>[]) new ImmutableSet<?>[args];
 			for (var i = 0; i < args; i++) {
-				var parameter = union(method, argValues.get(new ArgRef(method, i)));
+				var parameter = union(method, i, argValues.get(new ArgRef(method, i)));
 				if (parameter != null) {
 					allUnknown = false;
 				}