From 6b247cb0e18235f9a97d1d3e6a2e031f5bacefc8 Mon Sep 17 00:00:00 2001
From: Graham <gpe@openrs2.dev>
Date: Sun, 12 Jan 2020 19:15:02 +0000
Subject: [PATCH] Add ByteBuf extension methods for RSA encryption and
 decryption

---
 common/pom.xml                                |  4 ++
 .../java/dev/openrs2/common/crypto/Rsa.kt     | 26 ++++++++++
 .../java/dev/openrs2/common/crypto/RsaTest.kt | 47 +++++++++++++++++++
 pom.xml                                       |  5 ++
 4 files changed, 82 insertions(+)

diff --git a/common/pom.xml b/common/pom.xml
index 6715f805..114fac17 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -23,6 +23,10 @@
       <groupId>com.michael-bull.kotlin-inline-logger</groupId>
       <artifactId>kotlin-inline-logger-jvm</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-buffer</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.bouncycastle</groupId>
       <artifactId>bcprov-jdk15on</artifactId>
diff --git a/common/src/main/java/dev/openrs2/common/crypto/Rsa.kt b/common/src/main/java/dev/openrs2/common/crypto/Rsa.kt
index fa27e02d..964c8904 100644
--- a/common/src/main/java/dev/openrs2/common/crypto/Rsa.kt
+++ b/common/src/main/java/dev/openrs2/common/crypto/Rsa.kt
@@ -1,5 +1,7 @@
 package dev.openrs2.common.crypto
 
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.Unpooled
 import org.bouncycastle.asn1.DERNull
 import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers
 import org.bouncycastle.asn1.pkcs.PrivateKeyInfo
@@ -22,6 +24,30 @@ import java.math.BigInteger
 import java.nio.file.Files
 import java.nio.file.Path
 
+private fun ByteBuf.toBigInteger(): BigInteger {
+    val bytes: ByteArray
+    if (hasArray() && arrayOffset() == 0 && readerIndex() == 0 && readableBytes() == array().size) {
+        bytes = array()
+    } else {
+        bytes = ByteArray(readableBytes())
+        getBytes(readerIndex(), bytes)
+    }
+
+    return BigInteger(bytes)
+}
+
+private fun BigInteger.toByteBuf(): ByteBuf {
+    return Unpooled.wrappedBuffer(toByteArray())
+}
+
+fun ByteBuf.rsaEncrypt(key: RSAKeyParameters): ByteBuf {
+    return Rsa.encrypt(toBigInteger(), key).toByteBuf()
+}
+
+fun ByteBuf.rsaDecrypt(key: RSAKeyParameters): ByteBuf {
+    return Rsa.decrypt(toBigInteger(), key).toByteBuf()
+}
+
 object Rsa {
     private const val PUBLIC_KEY = "PUBLIC KEY"
     private const val PRIVATE_KEY = "PRIVATE KEY"
diff --git a/common/src/test/java/dev/openrs2/common/crypto/RsaTest.kt b/common/src/test/java/dev/openrs2/common/crypto/RsaTest.kt
index 24093549..c4b63176 100644
--- a/common/src/test/java/dev/openrs2/common/crypto/RsaTest.kt
+++ b/common/src/test/java/dev/openrs2/common/crypto/RsaTest.kt
@@ -1,5 +1,6 @@
 package dev.openrs2.common.crypto
 
+import io.netty.buffer.Unpooled
 import org.bouncycastle.crypto.params.RSAKeyParameters
 import org.bouncycastle.crypto.params.RSAPrivateCrtKeyParameters
 import org.bouncycastle.util.Properties
@@ -54,6 +55,52 @@ object RsaTest {
         assertEquals(BigInteger("65"), ciphertext)
     }
 
+    @Test
+    fun testEncryptByteBuf() {
+        // from https://en.wikipedia.org/wiki/RSA_(cryptosystem)#Example
+        val public = allowUnsafeMod { RSAKeyParameters(false, BigInteger("3233"), BigInteger("17")) }
+
+        val plaintext = Unpooled.wrappedBuffer(byteArrayOf(65))
+        try {
+            val ciphertext = plaintext.rsaEncrypt(public)
+            try {
+                val expectedCiphertext = Unpooled.wrappedBuffer(byteArrayOf(10, 230.toByte()))
+                try {
+                    assertEquals(expectedCiphertext, ciphertext)
+                } finally {
+                    expectedCiphertext.release()
+                }
+            } finally {
+                ciphertext.release()
+            }
+        } finally {
+            plaintext.release()
+        }
+    }
+
+    @Test
+    fun testDecryptByteBuf() {
+        // from https://en.wikipedia.org/wiki/RSA_(cryptosystem)#Example
+        val private = allowUnsafeMod { RSAKeyParameters(true, BigInteger("3233"), BigInteger("413")) }
+
+        val ciphertext = Unpooled.wrappedBuffer(byteArrayOf(10, 230.toByte()))
+        try {
+            val plaintext = ciphertext.rsaDecrypt(private)
+            try {
+                val expectedPlaintext = Unpooled.wrappedBuffer(byteArrayOf(65))
+                try {
+                    assertEquals(expectedPlaintext, plaintext)
+                } finally {
+                    expectedPlaintext.release()
+                }
+            } finally {
+                plaintext.release()
+            }
+        } finally {
+            ciphertext.release()
+        }
+    }
+
     private fun <T> allowUnsafeMod(f: () -> T): T {
         Properties.setThreadOverride(ALLOW_UNSAFE_MOD, true)
         try {
diff --git a/pom.xml b/pom.xml
index 336e1eb8..759f6111 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,6 +83,11 @@
         <artifactId>pack200</artifactId>
         <version>1.0.2</version>
       </dependency>
+      <dependency>
+        <groupId>io.netty</groupId>
+        <artifactId>netty-buffer</artifactId>
+        <version>4.1.44.Final</version>
+      </dependency>
       <dependency>
         <groupId>org.bouncycastle</groupId>
         <artifactId>bcprov-jdk15on</artifactId>