Fix XTEA en/decryption if the index is not a multiple of the block size

This commit updates the tests to ensure all combinations of 0-7 bytes of
leading/trailing blocks of unencrypted bytes work correctly with all
test vectors.

Signed-off-by: Graham <gpe@openrs2.dev>

crypto/src/main/java/dev/openrs2/crypto/Xtea.kt

@@ -10,7 +10,7 @@ private const val BLOCK_SIZE_MASK = BLOCK_SIZE - 1
 fun ByteBuf.xteaEncrypt(index: Int, length: Int, key: IntArray) {
     require(key.size == 4)
 
-    val end = (index + length) and BLOCK_SIZE_MASK.inv()
+    val end = index + (length and BLOCK_SIZE_MASK.inv())
     for (i in index until end step BLOCK_SIZE) {
         var sum = 0
         var v0 = getInt(i)
@@ -30,7 +30,7 @@ fun ByteBuf.xteaEncrypt(index: Int, length: Int, key: IntArray) {
 fun ByteBuf.xteaDecrypt(index: Int, length: Int, key: IntArray) {
     require(key.size == 4)
 
-    val end = (index + length) and BLOCK_SIZE_MASK.inv()
+    val end = index + (length and BLOCK_SIZE_MASK.inv())
     for (i in index until end step BLOCK_SIZE) {
         @Suppress("INTEGER_OVERFLOW")
         var sum = GOLDEN_RATIO * ROUNDS

crypto/src/test/java/dev/openrs2/crypto/XteaTest.kt

@@ -28,24 +28,22 @@ object XteaTest {
 
         // two blocks
         TestVector("00000000000000000000000000000000", "70e1225d6e4e76554141414141414141",
-            "4141414141414141ed23375a821a8c2d"),
-
-        // not a multiple of the block size
-        TestVector("00000000000000000000000000000000", "01", "01"),
-        TestVector("00000000000000000000000000000000", "01020304050607", "01020304050607"),
-        TestVector("00000000000000000000000000000000", "70e1225d6e4e765501", "414141414141414101"),
-        TestVector("00000000000000000000000000000000", "70e1225d6e4e765501020304050607",
-            "414141414141414101020304050607")
+            "4141414141414141ed23375a821a8c2d")
     )
 
     @Test
     fun testEncrypt() {
         for (vector in TEST_VECTORS) {
-            val buffer = Unpooled.copiedBuffer(vector.plaintext)
+            for (i in 0..7) {
+                for (j in 0..7) {
+                    val header = ByteArray(i) { it.toByte() }
+                    val trailer = ByteArray(j) { it.toByte() }
+
+                    val buffer = Unpooled.copiedBuffer(header, vector.plaintext, trailer)
                     try {
-                buffer.xteaEncrypt(0, buffer.readableBytes(), vector.key)
+                        buffer.xteaEncrypt(i, vector.plaintext.size, vector.key)
 
-                val expected = Unpooled.wrappedBuffer(vector.ciphertext)
+                        val expected = Unpooled.wrappedBuffer(header, vector.ciphertext, trailer)
                         try {
                             assertEquals(expected, buffer)
                         } finally {
@@ -56,15 +54,22 @@ object XteaTest {
                     }
                 }
             }
+        }
+    }
 
     @Test
     fun testDecrypt() {
         for (vector in TEST_VECTORS) {
-            val buffer = Unpooled.copiedBuffer(vector.ciphertext)
+            for (i in 0..7) {
+                for (j in 0..7) {
+                    val header = ByteArray(i) { it.toByte() }
+                    val trailer = ByteArray(j) { it.toByte() }
+
+                    val buffer = Unpooled.copiedBuffer(header, vector.ciphertext, trailer)
                     try {
-                buffer.xteaDecrypt(0, buffer.readableBytes(), vector.key)
+                        buffer.xteaDecrypt(i, vector.ciphertext.size, vector.key)
 
-                val expected = Unpooled.wrappedBuffer(vector.plaintext)
+                        val expected = Unpooled.wrappedBuffer(header, vector.plaintext, trailer)
                         try {
                             assertEquals(expected, buffer)
                         } finally {
@@ -76,3 +81,5 @@ object XteaTest {
                 }
             }
         }
+    }
+}