Add JS5 compression/encryption implementation

Signed-off-by: Graham <gpe@openrs2.dev>

cache/build.gradle.kts

@@ -0,0 +1,26 @@
+plugins {
+    `maven-publish`
+    kotlin("jvm")
+}
+
+dependencies {
+    implementation(project(":buffer"))
+    implementation(project(":compress"))
+    implementation(project(":crypto"))
+}
+
+publishing {
+    publications.create<MavenPublication>("maven") {
+        from(components["java"])
+
+        pom {
+            packaging = "jar"
+            name.set("OpenRS2 Cache")
+            description.set(
+                """
+                A library for reading and writing the RuneScape cache.
+            """.trimIndent()
+            )
+        }
+    }
+}

cache/src/main/java/dev/openrs2/cache/Js5Compression.kt

@@ -0,0 +1,126 @@
+package dev.openrs2.cache
+
+import dev.openrs2.buffer.use
+import dev.openrs2.crypto.XteaKey
+import dev.openrs2.crypto.xteaDecrypt
+import dev.openrs2.crypto.xteaEncrypt
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufInputStream
+import io.netty.buffer.ByteBufOutputStream
+
+object Js5Compression {
+    fun compress(input: ByteBuf, type: Js5CompressionType, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        input.alloc().buffer().use { output ->
+            output.writeByte(type.ordinal)
+
+            if (type == Js5CompressionType.NONE) {
+                val len = input.readableBytes()
+                output.writeInt(len)
+                output.writeBytes(input)
+
+                if (!key.isZero) {
+                    output.xteaEncrypt(5, len, key)
+                }
+
+                return output.retain()
+            }
+
+            val lenIndex = output.writerIndex()
+            output.writeZero(4)
+
+            output.writeInt(input.readableBytes())
+
+            val start = output.writerIndex()
+
+            type.createOutputStream(ByteBufOutputStream(output)).use { outputStream ->
+                ByteBufInputStream(input).use { inputStream ->
+                    inputStream.copyTo(outputStream)
+                }
+            }
+
+            val len = output.writerIndex() - start
+            output.setInt(lenIndex, len)
+
+            if (!key.isZero) {
+                output.xteaEncrypt(5, len + 4, key)
+            }
+
+            return output.retain()
+        }
+    }
+
+    fun compressBest(input: ByteBuf, enableLzma: Boolean = false, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        var best = compress(input.slice(), Js5CompressionType.NONE, key)
+        try {
+            for (type in Js5CompressionType.values()) {
+                if (type == Js5CompressionType.NONE || (type == Js5CompressionType.LZMA && !enableLzma)) {
+                    continue
+                }
+
+                compress(input.slice(), type, key).use { output ->
+                    if (output.readableBytes() < best.readableBytes()) {
+                        best.release()
+                        best = output.retain()
+                    }
+                }
+            }
+
+            // consume all of input so this method is a drop-in replacement for compress()
+            input.skipBytes(input.readableBytes())
+
+            return best.retain()
+        } finally {
+            best.release()
+        }
+    }
+
+    fun uncompress(input: ByteBuf, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        val typeId = input.readUnsignedByte().toInt()
+        val type = Js5CompressionType.fromOrdinal(typeId)
+        require(type != null) {
+            "Invalid compression type: $typeId"
+        }
+
+        val len = input.readInt()
+        require(len >= 0) {
+            "Length is negative: $len"
+        }
+
+        if (type == Js5CompressionType.NONE) {
+            input.readBytes(len).use { output ->
+                if (!key.isZero) {
+                    output.xteaDecrypt(0, len, key)
+                }
+                return output.retain()
+            }
+        }
+
+        decrypt(input, len + 4, key).use { plaintext ->
+            val uncompressedLen = plaintext.readInt()
+            require(uncompressedLen >= 0) {
+                "Uncompressed length is negative: $uncompressedLen"
+            }
+
+            plaintext.alloc().buffer(uncompressedLen, uncompressedLen).use { output ->
+                type.createInputStream(ByteBufInputStream(plaintext, len), uncompressedLen).use { inputStream ->
+                    ByteBufOutputStream(output).use { outputStream ->
+                        inputStream.copyTo(outputStream)
+                    }
+                }
+
+                return output.retain()
+            }
+        }
+    }
+
+    private fun decrypt(buf: ByteBuf, len: Int, key: XteaKey): ByteBuf {
+        if (key.isZero) {
+            return buf.readRetainedSlice(len)
+        }
+
+        buf.readBytes(len).use { output ->
+            output.xteaDecrypt(0, len, key)
+            return output.retain()
+        }
+    }
+}

cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt

@@ -0,0 +1,51 @@
+package dev.openrs2.cache
+
+import dev.openrs2.compress.bzip2.Bzip2
+import dev.openrs2.compress.gzip.Gzip
+import dev.openrs2.compress.lzma.Lzma
+import java.io.InputStream
+import java.io.OutputStream
+import java.util.zip.Deflater
+
+enum class Js5CompressionType {
+    NONE,
+    BZIP2,
+    GZIP,
+    LZMA;
+
+    fun createInputStream(input: InputStream, length: Int): InputStream {
+        return when (this) {
+            NONE -> input
+            BZIP2 -> Bzip2.createHeaderlessInputStream(input)
+            GZIP -> Gzip.createHeaderlessInputStream(input)
+            LZMA -> Lzma.createHeaderlessInputStream(input, length.toLong())
+        }
+    }
+
+    fun createOutputStream(output: OutputStream): OutputStream {
+        return when (this) {
+            NONE -> output
+            BZIP2 -> Bzip2.createHeaderlessOutputStream(output)
+            GZIP -> Gzip.createHeaderlessOutputStream(output, Deflater.BEST_COMPRESSION)
+            /*
+             * LZMA at -9 has significantly higher CPU/memory requirements for
+             * both compression _and_ decompression, so we use the default of
+             * -6. Using a higher level for the typical file size in the
+             * RuneScape cache probably provides insignificant returns, as
+             * described in the LZMA documentation.
+             */
+            LZMA -> Lzma.createHeaderlessOutputStream(output, Lzma.DEFAULT_COMPRESSION)
+        }
+    }
+
+    companion object {
+        fun fromOrdinal(ordinal: Int): Js5CompressionType? {
+            val values = values()
+            return if (ordinal >= 0 && ordinal < values.size) {
+                values[ordinal]
+            } else {
+                null
+            }
+        }
+    }
+}

cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt

@@ -0,0 +1,231 @@
+package dev.openrs2.cache
+
+import dev.openrs2.buffer.use
+import dev.openrs2.crypto.XteaKey
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.Unpooled
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
+
+object Js5CompressionTest {
+    private val KEY = XteaKey(intArrayOf(0x00112233, 0x44556677, 0x8899AABB.toInt(), 0xCCDDEEFF.toInt()))
+
+    @Test
+    fun testCompressNone() {
+        read("none.dat").use { expected ->
+            Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { input ->
+                Js5Compression.compress(input, Js5CompressionType.NONE).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressNone() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("none.dat").use { input ->
+                Js5Compression.uncompress(input).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressGzip() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP).use { compressed ->
+                Js5Compression.uncompress(compressed).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressGzip() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("gzip.dat").use { input ->
+                Js5Compression.uncompress(input).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressBzip2() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2).use { compressed ->
+                Js5Compression.uncompress(compressed).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressBzip2() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("bzip2.dat").use { input ->
+                Js5Compression.uncompress(input).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressLzma() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA).use { compressed ->
+                Js5Compression.uncompress(compressed).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressLzma() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("lzma.dat").use { input ->
+                Js5Compression.uncompress(input).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressNoneEncrypted() {
+        read("none-encrypted.dat").use { expected ->
+            Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { input ->
+                Js5Compression.compress(input, Js5CompressionType.NONE, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressNoneEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { expected ->
+            read("none-encrypted.dat").use { input ->
+                Js5Compression.uncompress(input, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressGzipEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP, KEY).use { compressed ->
+                Js5Compression.uncompress(compressed, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressGzipEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("gzip-encrypted.dat").use { input ->
+                Js5Compression.uncompress(input, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressBzip2Encrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2, KEY).use { compressed ->
+                Js5Compression.uncompress(compressed, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressBzip2Encrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("bzip2-encrypted.dat").use { input ->
+                Js5Compression.uncompress(input, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressLzmaEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA, KEY).use { compressed ->
+                Js5Compression.uncompress(compressed, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testUncompressLzmaEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
+            read("lzma-encrypted.dat").use { input ->
+                Js5Compression.uncompress(input, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressBest() {
+        Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected ->
+            val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed ->
+                compressed.readableBytes()
+            }
+
+            Js5Compression.compressBest(expected.slice()).use { compressed ->
+                assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt())
+                assert(compressed.readableBytes() < noneLen)
+
+                Js5Compression.uncompress(compressed).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    @Test
+    fun testCompressBestEncrypted() {
+        Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected ->
+            val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed ->
+                compressed.readableBytes()
+            }
+
+            Js5Compression.compressBest(expected.slice(), key = KEY).use { compressed ->
+                assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt())
+                assert(compressed.readableBytes() < noneLen)
+
+                Js5Compression.uncompress(compressed, KEY).use { actual ->
+                    assertEquals(expected, actual)
+                }
+            }
+        }
+    }
+
+    private fun read(name: String): ByteBuf {
+        Js5CompressionTest::class.java.getResourceAsStream(name).use { input ->
+            return Unpooled.wrappedBuffer(input.readAllBytes())
+        }
+    }
+}

settings.gradle.kts

@@ -5,6 +5,7 @@ include(
     "asm",
     "buffer",
     "bundler",
+    "cache",
     "compress",
     "compress-cli",
     "conf",