From 012d5ed0db8ce639bf515ef25947e12edb096cfe Mon Sep 17 00:00:00 2001
From: Graham <gpe@openrs2.dev>
Date: Fri, 16 Aug 2019 22:10:01 +0100
Subject: [PATCH] Simplify combined shift/mask operations

---
 .../dev/openrs2/deob/ast/AstDeobfuscator.java |  2 +
 .../ast/transform/BitMaskTransformer.java     | 77 +++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 deob-ast/src/main/java/dev/openrs2/deob/ast/transform/BitMaskTransformer.java

diff --git a/deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscator.java b/deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscator.java
index 1b7ff6bcd8..6e2c983742 100644
--- a/deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscator.java
+++ b/deob-ast/src/main/java/dev/openrs2/deob/ast/AstDeobfuscator.java
@@ -14,6 +14,7 @@ import com.github.javaparser.utils.SourceRoot;
 import com.google.common.collect.ImmutableList;
 import dev.openrs2.deob.ast.transform.AddSubTransformer;
 import dev.openrs2.deob.ast.transform.BinaryExprOrderTransformer;
+import dev.openrs2.deob.ast.transform.BitMaskTransformer;
 import dev.openrs2.deob.ast.transform.ComplementTransformer;
 import dev.openrs2.deob.ast.transform.EncloseTransformer;
 import dev.openrs2.deob.ast.transform.IfElseTransformer;
@@ -31,6 +32,7 @@ public final class AstDeobfuscator {
 		new TernaryTransformer(),
 		new BinaryExprOrderTransformer(),
 		new AddSubTransformer(),
+		new BitMaskTransformer(),
 		new EncloseTransformer()
 	);
 
diff --git a/deob-ast/src/main/java/dev/openrs2/deob/ast/transform/BitMaskTransformer.java b/deob-ast/src/main/java/dev/openrs2/deob/ast/transform/BitMaskTransformer.java
new file mode 100644
index 0000000000..f68569ef2f
--- /dev/null
+++ b/deob-ast/src/main/java/dev/openrs2/deob/ast/transform/BitMaskTransformer.java
@@ -0,0 +1,77 @@
+package dev.openrs2.deob.ast.transform;
+
+import com.github.javaparser.ast.CompilationUnit;
+import com.github.javaparser.ast.expr.BinaryExpr;
+import com.github.javaparser.ast.expr.IntegerLiteralExpr;
+import com.google.common.collect.ImmutableSet;
+import dev.openrs2.deob.ast.util.ExprUtils;
+
+public final class BitMaskTransformer extends Transformer {
+	private static final ImmutableSet<BinaryExpr.Operator> SHIFT_OPS = ImmutableSet.of(
+		BinaryExpr.Operator.SIGNED_RIGHT_SHIFT,
+		BinaryExpr.Operator.UNSIGNED_RIGHT_SHIFT
+	);
+
+	private static final ImmutableSet<BinaryExpr.Operator> BITWISE_OPS = ImmutableSet.of(
+		BinaryExpr.Operator.BINARY_AND,
+		BinaryExpr.Operator.BINARY_OR,
+		BinaryExpr.Operator.XOR
+	);
+
+	@Override
+	public void transform(CompilationUnit unit) {
+		unit.findAll(BinaryExpr.class).forEach(expr -> {
+			var shiftOp = expr.getOperator();
+			var left = expr.getLeft();
+			var shamtExpr = expr.getRight();
+
+			if (!SHIFT_OPS.contains(shiftOp) || !left.isBinaryExpr() || !shamtExpr.isIntegerLiteralExpr()) {
+				return;
+			}
+
+			var bitwiseExpr = left.asBinaryExpr();
+			var bitwiseOp = bitwiseExpr.getOperator();
+			var argExpr = bitwiseExpr.getLeft();
+			var maskExpr = bitwiseExpr.getRight();
+
+			if (!BITWISE_OPS.contains(bitwiseOp) || !ExprUtils.isIntegerOrLongLiteral(maskExpr)) {
+				return;
+			}
+
+			var shamt = shamtExpr.asIntegerLiteralExpr().asInt();
+			if (maskExpr.isIntegerLiteralExpr()) {
+				var mask = maskExpr.asIntegerLiteralExpr().asInt();
+
+				switch (shiftOp) {
+				case SIGNED_RIGHT_SHIFT:
+					mask >>= shamt;
+					break;
+				case UNSIGNED_RIGHT_SHIFT:
+					mask >>>= shamt;
+					break;
+				default:
+					throw new IllegalStateException();
+				}
+
+				maskExpr = new IntegerLiteralExpr(mask);
+			} else {
+				var mask = maskExpr.asLongLiteralExpr().asLong();
+
+				switch (shiftOp) {
+				case SIGNED_RIGHT_SHIFT:
+					mask >>= shamt;
+					break;
+				case UNSIGNED_RIGHT_SHIFT:
+					mask >>>= shamt;
+					break;
+				default:
+					throw new IllegalStateException();
+				}
+
+				maskExpr = ExprUtils.createLong(mask);
+			}
+
+			expr.replace(new BinaryExpr(new BinaryExpr(argExpr.clone(), shamtExpr.clone(), shiftOp), maskExpr, bitwiseOp));
+		});
+	}
+}