Add functions for quickly checking if a XTEA key is valid

Signed-off-by: Graham <gpe@openrs2.dev>

cache/src/main/java/dev/openrs2/cache/Js5Compression.kt

@@ -8,8 +8,17 @@ import io.netty.buffer.ByteBuf
 import io.netty.buffer.ByteBufInputStream
 import io.netty.buffer.ByteBufOutputStream
 import java.io.IOException
+import java.io.OutputStream
 
 public object Js5Compression {
+    private val BZIP2_MAGIC = byteArrayOf(0x31, 0x41, 0x59, 0x26, 0x53, 0x59)
+
+    private const val GZIP_MAGIC = 0x1F8B
+    private const val GZIP_COMPRESSION_METHOD_DEFLATE = 0x08
+
+    private const val LZMA_PB_MAX = 4
+    private const val LZMA_PRESET_DICT_SIZE_MAX = 1 shl 26
+
     public fun compress(input: ByteBuf, type: Js5CompressionType, key: XteaKey = XteaKey.ZERO): ByteBuf {
         input.alloc().buffer().use { output ->
             output.writeByte(type.ordinal)
@@ -150,6 +159,153 @@ public object Js5Compression {
         }
     }
 
+    public fun isEncrypted(input: ByteBuf): Boolean {
+        return !isKeyValid(input, XteaKey.ZERO)
+    }
+
+    public fun isKeyValid(input: ByteBuf, key: XteaKey): Boolean {
+        val typeId = input.readUnsignedByte().toInt()
+        val type = Js5CompressionType.fromOrdinal(typeId)
+            ?: throw IOException("Invalid compression type: $typeId")
+
+        val len = input.readInt()
+        if (len < 0) {
+            throw IOException("Length is negative: $len")
+        }
+
+        if (type == Js5CompressionType.NONE) {
+            /*
+             * There is no easy way for us to be sure whether an uncompressed
+             * group's key is valid or not, as we'd need specific validation
+             * code for each file format the client uses (currently only maps,
+             * though apparently RS3 is also capable of encrypting interfaces).
+             *
+             * In practice, encrypted files don't tend to be uncompressed
+             * anyway - in 550, this functionality is actually broken due to a
+             * bug (see the comment about enableUncompressedEncryption in
+             * compressBest above).
+             *
+             * We therefore assume all uncompressed groups are unencrypted.
+             */
+            return key.isZero
+        }
+
+        val lenWithUncompressedLen = len + 4
+        if (input.readableBytes() < lenWithUncompressedLen) {
+            throw IOException("Compressed data truncated")
+        }
+
+        /*
+         * Decrypt two XTEA blocks, which is sufficient to quickly check the
+         * uncompressed length and the compression algorithm's header in each
+         * case:
+         *
+         * BZIP2: 6 byte block magic
+         * GZIP:  2 byte magic, 1 byte compression method
+         * LZMA:  1 byte properties, 4 byte dictionary size
+         *
+         * We never expect to see a file with fewer than two blocks. The
+         * compressed length of an empty file is always two XTEA blocks in each
+         * case:
+         *
+         * BZIP2: 10 bytes
+         * GZIP:  18 bytes
+         * LZMA:  10 bytes
+         */
+        if (lenWithUncompressedLen < 16) {
+            throw IOException("Compressed data shorter than two XTEA blocks")
+        }
+
+        decrypt(input.slice(), 16, key).use { plaintext ->
+            val uncompressedLen = plaintext.readInt()
+            if (uncompressedLen < 0) {
+                return false
+            }
+
+            when (type) {
+                Js5CompressionType.NONE -> throw AssertionError()
+                Js5CompressionType.BZIP2 -> {
+                    val magic = ByteArray(BZIP2_MAGIC.size)
+                    plaintext.readBytes(magic)
+                    if (!magic.contentEquals(BZIP2_MAGIC)) {
+                        return false
+                    }
+                }
+                Js5CompressionType.GZIP -> {
+                    val magic = plaintext.readUnsignedShort()
+                    if (magic != GZIP_MAGIC) {
+                        return false
+                    }
+
+                    // Jagex's implementation only supports DEFLATE.
+                    val compressionMethod = plaintext.readUnsignedByte().toInt()
+                    if (compressionMethod != GZIP_COMPRESSION_METHOD_DEFLATE) {
+                        return false
+                    }
+                }
+                Js5CompressionType.LZMA -> {
+                    val properties = plaintext.readUnsignedByte()
+
+                    /*
+                     * The encoding of the properties byte means it isn't
+                     * possible for lc/lp to be out of range.
+                     */
+
+                    val pb = properties / 45
+                    if (pb > LZMA_PB_MAX) {
+                        return false
+                    }
+
+                    /*
+                     * Jagex's implementation doesn't support dictionary sizes
+                     * greater than 2 GiB (which are less than zero when the
+                     * dictionary size is read as a signed integer).
+                     *
+                     * We also assume dictionary sizes larger than 64 MiB (the
+                     * size of the dictionary at -9, the highest preset level)
+                     * indicate the key is incorrect.
+                     *
+                     * In theory these dictionary sizes are actually valid.
+                     * However, in practice Jagex seems to only use level -3 or
+                     * -4 (as their dictionary size is 4 MiB).
+                     *
+                     * Attempting to decompress LZMA streams with larger
+                     * dictionaries can easily cause OOM exceptions in low
+                     * memory environments.
+                     */
+                    val dictSize = plaintext.readIntLE()
+                    if (dictSize < 0) {
+                        return false
+                    } else if (dictSize > LZMA_PRESET_DICT_SIZE_MAX) {
+                        return false
+                    }
+                }
+            }
+        }
+
+        // Run the entire decompression algorithm to confirm the key is valid.
+        decrypt(input, lenWithUncompressedLen, key).use { plaintext ->
+            val uncompressedLen = plaintext.readInt()
+            if (uncompressedLen < 0) {
+                throw AssertionError()
+            }
+
+            try {
+                OutputStream.nullOutputStream().use { output ->
+                    type.createInputStream(ByteBufInputStream(plaintext, len), uncompressedLen).use { inputStream ->
+                        if (inputStream.transferTo(output) != uncompressedLen.toLong()) {
+                            return false
+                        }
+                    }
+                }
+            } catch (ex: IOException) {
+                return false
+            }
+        }
+
+        return true
+    }
+
     private fun decrypt(buf: ByteBuf, len: Int, key: XteaKey): ByteBuf {
         if (key.isZero) {
             return buf.readRetainedSlice(len)

cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt

@@ -8,10 +8,13 @@ import org.junit.jupiter.api.assertThrows
 import java.io.IOException
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
 import kotlin.test.assertNotEquals
+import kotlin.test.assertTrue
 
 object Js5CompressionTest {
     private val KEY = XteaKey.fromHex("00112233445566778899AABBCCDDEEFF")
+    private val INVALID_KEY = XteaKey.fromHex("0123456789ABCDEF0123456789ABCDEF")
 
     @Test
     fun testCompressNone() {
@@ -261,7 +264,11 @@ object Js5CompressionTest {
     fun testInvalidType() {
         read("invalid-type.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
+            }
+
+            assertThrows<IOException> {
+                Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO)
             }
         }
     }
@@ -270,7 +277,11 @@ object Js5CompressionTest {
     fun testInvalidLength() {
         read("invalid-length.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
+            }
+
+            assertThrows<IOException> {
+                Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO)
             }
         }
     }
@@ -279,8 +290,10 @@ object Js5CompressionTest {
     fun testInvalidUncompressedLength() {
         read("invalid-uncompressed-length.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -297,8 +310,10 @@ object Js5CompressionTest {
     fun testBzip2Eof() {
         read("bzip2-eof.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -306,8 +321,10 @@ object Js5CompressionTest {
     fun testGzipEof() {
         read("gzip-eof.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -315,8 +332,10 @@ object Js5CompressionTest {
     fun testLzmaEof() {
         read("lzma-eof.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -324,8 +343,10 @@ object Js5CompressionTest {
     fun testBzip2Corrupt() {
         read("bzip2-corrupt.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -333,8 +354,10 @@ object Js5CompressionTest {
     fun testGzipCorrupt() {
         read("gzip-corrupt.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
             }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
         }
     }
 
@@ -342,7 +365,103 @@ object Js5CompressionTest {
     fun testLzmaCorrupt() {
         read("lzma-corrupt.dat").use { compressed ->
             assertThrows<IOException> {
-                Js5Compression.uncompress(compressed).release()
+                Js5Compression.uncompress(compressed.slice()).release()
+            }
+
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+        }
+    }
+
+    @Test
+    fun testNoneKeyValid() {
+        read("none.dat").use { compressed ->
+            assertFalse(Js5Compression.isEncrypted(compressed.slice()))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+    }
+
+    @Test
+    fun testBzip2KeyValid() {
+        read("bzip2.dat").use { compressed ->
+            assertFalse(Js5Compression.isEncrypted(compressed.slice()))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("bzip2-encrypted.dat").use { compressed ->
+            assertTrue(Js5Compression.isEncrypted(compressed.slice()))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("bzip2-invalid-magic.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+    }
+
+    @Test
+    fun testGzipKeyValid() {
+        read("gzip.dat").use { compressed ->
+            assertFalse(Js5Compression.isEncrypted(compressed.slice()))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("gzip-encrypted.dat").use { compressed ->
+            assertTrue(Js5Compression.isEncrypted(compressed.slice()))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("gzip-invalid-magic.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+
+        read("gzip-invalid-method.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+    }
+
+    @Test
+    fun testLzmaKeyValid() {
+        read("lzma.dat").use { compressed ->
+            assertFalse(Js5Compression.isEncrypted(compressed.slice()))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("lzma-encrypted.dat").use { compressed ->
+            assertTrue(Js5Compression.isEncrypted(compressed.slice()))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), XteaKey.ZERO))
+            assertTrue(Js5Compression.isKeyValid(compressed.slice(), KEY))
+            assertFalse(Js5Compression.isKeyValid(compressed.slice(), INVALID_KEY))
+        }
+
+        read("lzma-dict-size-negative.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+
+        read("lzma-dict-size-larger-than-preset.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+
+        read("lzma-invalid-pb.dat").use { compressed ->
+            assertFalse(Js5Compression.isKeyValid(compressed, XteaKey.ZERO))
+        }
+    }
+
+    @Test
+    fun testKeyValidShorterThanTwoBlocks() {
+        read("shorter-than-two-blocks.dat").use { compressed ->
+            assertThrows<IOException> {
+                Js5Compression.isKeyValid(compressed, XteaKey.ZERO)
             }
         }
     }