From 58335ca6d0bfeb90ccc1a271a333c29d7b57d12e Mon Sep 17 00:00:00 2001 From: Graham Date: Wed, 12 Aug 2020 21:57:48 +0100 Subject: [PATCH] Add JS5 compression/encryption implementation Signed-off-by: Graham --- cache/build.gradle.kts | 26 ++ .../java/dev/openrs2/cache/Js5Compression.kt | 126 ++++++++++ .../dev/openrs2/cache/Js5CompressionType.kt | 51 ++++ .../dev/openrs2/cache/Js5CompressionTest.kt | 231 ++++++++++++++++++ .../dev/openrs2/cache/bzip2-encrypted.dat | Bin 0 -> 55 bytes .../resources/dev/openrs2/cache/bzip2.dat | Bin 0 -> 55 bytes .../dev/openrs2/cache/gzip-encrypted.dat | Bin 0 -> 34 bytes .../test/resources/dev/openrs2/cache/gzip.dat | Bin 0 -> 34 bytes .../dev/openrs2/cache/lzma-encrypted.dat | Bin 0 -> 26 bytes .../test/resources/dev/openrs2/cache/lzma.dat | Bin 0 -> 26 bytes .../dev/openrs2/cache/none-encrypted.dat | Bin 0 -> 26 bytes .../test/resources/dev/openrs2/cache/none.dat | Bin 0 -> 12 bytes settings.gradle.kts | 1 + 13 files changed, 435 insertions(+) create mode 100644 cache/build.gradle.kts create mode 100644 cache/src/main/java/dev/openrs2/cache/Js5Compression.kt create mode 100644 cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt create mode 100644 cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt create mode 100644 cache/src/test/resources/dev/openrs2/cache/bzip2-encrypted.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/bzip2.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/gzip-encrypted.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/gzip.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/lzma-encrypted.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/lzma.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/none-encrypted.dat create mode 100644 cache/src/test/resources/dev/openrs2/cache/none.dat diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts new file mode 100644 index 0000000000..6978df90c1 --- /dev/null +++ b/cache/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + `maven-publish` + kotlin("jvm") +} + +dependencies { + implementation(project(":buffer")) + implementation(project(":compress")) + implementation(project(":crypto")) +} + +publishing { + publications.create("maven") { + from(components["java"]) + + pom { + packaging = "jar" + name.set("OpenRS2 Cache") + description.set( + """ + A library for reading and writing the RuneScape cache. + """.trimIndent() + ) + } + } +} diff --git a/cache/src/main/java/dev/openrs2/cache/Js5Compression.kt b/cache/src/main/java/dev/openrs2/cache/Js5Compression.kt new file mode 100644 index 0000000000..6bdf044ba7 --- /dev/null +++ b/cache/src/main/java/dev/openrs2/cache/Js5Compression.kt @@ -0,0 +1,126 @@ +package dev.openrs2.cache + +import dev.openrs2.buffer.use +import dev.openrs2.crypto.XteaKey +import dev.openrs2.crypto.xteaDecrypt +import dev.openrs2.crypto.xteaEncrypt +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufInputStream +import io.netty.buffer.ByteBufOutputStream + +object Js5Compression { + fun compress(input: ByteBuf, type: Js5CompressionType, key: XteaKey = XteaKey.ZERO): ByteBuf { + input.alloc().buffer().use { output -> + output.writeByte(type.ordinal) + + if (type == Js5CompressionType.NONE) { + val len = input.readableBytes() + output.writeInt(len) + output.writeBytes(input) + + if (!key.isZero) { + output.xteaEncrypt(5, len, key) + } + + return output.retain() + } + + val lenIndex = output.writerIndex() + output.writeZero(4) + + output.writeInt(input.readableBytes()) + + val start = output.writerIndex() + + type.createOutputStream(ByteBufOutputStream(output)).use { outputStream -> + ByteBufInputStream(input).use { inputStream -> + inputStream.copyTo(outputStream) + } + } + + val len = output.writerIndex() - start + output.setInt(lenIndex, len) + + if (!key.isZero) { + output.xteaEncrypt(5, len + 4, key) + } + + return output.retain() + } + } + + fun compressBest(input: ByteBuf, enableLzma: Boolean = false, key: XteaKey = XteaKey.ZERO): ByteBuf { + var best = compress(input.slice(), Js5CompressionType.NONE, key) + try { + for (type in Js5CompressionType.values()) { + if (type == Js5CompressionType.NONE || (type == Js5CompressionType.LZMA && !enableLzma)) { + continue + } + + compress(input.slice(), type, key).use { output -> + if (output.readableBytes() < best.readableBytes()) { + best.release() + best = output.retain() + } + } + } + + // consume all of input so this method is a drop-in replacement for compress() + input.skipBytes(input.readableBytes()) + + return best.retain() + } finally { + best.release() + } + } + + fun uncompress(input: ByteBuf, key: XteaKey = XteaKey.ZERO): ByteBuf { + val typeId = input.readUnsignedByte().toInt() + val type = Js5CompressionType.fromOrdinal(typeId) + require(type != null) { + "Invalid compression type: $typeId" + } + + val len = input.readInt() + require(len >= 0) { + "Length is negative: $len" + } + + if (type == Js5CompressionType.NONE) { + input.readBytes(len).use { output -> + if (!key.isZero) { + output.xteaDecrypt(0, len, key) + } + return output.retain() + } + } + + decrypt(input, len + 4, key).use { plaintext -> + val uncompressedLen = plaintext.readInt() + require(uncompressedLen >= 0) { + "Uncompressed length is negative: $uncompressedLen" + } + + plaintext.alloc().buffer(uncompressedLen, uncompressedLen).use { output -> + type.createInputStream(ByteBufInputStream(plaintext, len), uncompressedLen).use { inputStream -> + ByteBufOutputStream(output).use { outputStream -> + inputStream.copyTo(outputStream) + } + } + + return output.retain() + } + } + } + + private fun decrypt(buf: ByteBuf, len: Int, key: XteaKey): ByteBuf { + if (key.isZero) { + return buf.readRetainedSlice(len) + } + + buf.readBytes(len).use { output -> + output.xteaDecrypt(0, len, key) + return output.retain() + } + } +} diff --git a/cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt b/cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt new file mode 100644 index 0000000000..99f3e11d38 --- /dev/null +++ b/cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt @@ -0,0 +1,51 @@ +package dev.openrs2.cache + +import dev.openrs2.compress.bzip2.Bzip2 +import dev.openrs2.compress.gzip.Gzip +import dev.openrs2.compress.lzma.Lzma +import java.io.InputStream +import java.io.OutputStream +import java.util.zip.Deflater + +enum class Js5CompressionType { + NONE, + BZIP2, + GZIP, + LZMA; + + fun createInputStream(input: InputStream, length: Int): InputStream { + return when (this) { + NONE -> input + BZIP2 -> Bzip2.createHeaderlessInputStream(input) + GZIP -> Gzip.createHeaderlessInputStream(input) + LZMA -> Lzma.createHeaderlessInputStream(input, length.toLong()) + } + } + + fun createOutputStream(output: OutputStream): OutputStream { + return when (this) { + NONE -> output + BZIP2 -> Bzip2.createHeaderlessOutputStream(output) + GZIP -> Gzip.createHeaderlessOutputStream(output, Deflater.BEST_COMPRESSION) + /* + * LZMA at -9 has significantly higher CPU/memory requirements for + * both compression _and_ decompression, so we use the default of + * -6. Using a higher level for the typical file size in the + * RuneScape cache probably provides insignificant returns, as + * described in the LZMA documentation. + */ + LZMA -> Lzma.createHeaderlessOutputStream(output, Lzma.DEFAULT_COMPRESSION) + } + } + + companion object { + fun fromOrdinal(ordinal: Int): Js5CompressionType? { + val values = values() + return if (ordinal >= 0 && ordinal < values.size) { + values[ordinal] + } else { + null + } + } + } +} diff --git a/cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt b/cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt new file mode 100644 index 0000000000..d44169f79f --- /dev/null +++ b/cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt @@ -0,0 +1,231 @@ +package dev.openrs2.cache + +import dev.openrs2.buffer.use +import dev.openrs2.crypto.XteaKey +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals + +object Js5CompressionTest { + private val KEY = XteaKey(intArrayOf(0x00112233, 0x44556677, 0x8899AABB.toInt(), 0xCCDDEEFF.toInt())) + + @Test + fun testCompressNone() { + read("none.dat").use { expected -> + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { input -> + Js5Compression.compress(input, Js5CompressionType.NONE).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressNone() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("none.dat").use { input -> + Js5Compression.uncompress(input).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressGzip() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP).use { compressed -> + Js5Compression.uncompress(compressed).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressGzip() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("gzip.dat").use { input -> + Js5Compression.uncompress(input).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressBzip2() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2).use { compressed -> + Js5Compression.uncompress(compressed).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressBzip2() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("bzip2.dat").use { input -> + Js5Compression.uncompress(input).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressLzma() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA).use { compressed -> + Js5Compression.uncompress(compressed).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressLzma() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("lzma.dat").use { input -> + Js5Compression.uncompress(input).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressNoneEncrypted() { + read("none-encrypted.dat").use { expected -> + Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { input -> + Js5Compression.compress(input, Js5CompressionType.NONE, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressNoneEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { expected -> + read("none-encrypted.dat").use { input -> + Js5Compression.uncompress(input, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressGzipEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP, KEY).use { compressed -> + Js5Compression.uncompress(compressed, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressGzipEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("gzip-encrypted.dat").use { input -> + Js5Compression.uncompress(input, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressBzip2Encrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2, KEY).use { compressed -> + Js5Compression.uncompress(compressed, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressBzip2Encrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("bzip2-encrypted.dat").use { input -> + Js5Compression.uncompress(input, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressLzmaEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA, KEY).use { compressed -> + Js5Compression.uncompress(compressed, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testUncompressLzmaEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected -> + read("lzma-encrypted.dat").use { input -> + Js5Compression.uncompress(input, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressBest() { + Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected -> + val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed -> + compressed.readableBytes() + } + + Js5Compression.compressBest(expected.slice()).use { compressed -> + assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt()) + assert(compressed.readableBytes() < noneLen) + + Js5Compression.uncompress(compressed).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + @Test + fun testCompressBestEncrypted() { + Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected -> + val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed -> + compressed.readableBytes() + } + + Js5Compression.compressBest(expected.slice(), key = KEY).use { compressed -> + assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt()) + assert(compressed.readableBytes() < noneLen) + + Js5Compression.uncompress(compressed, KEY).use { actual -> + assertEquals(expected, actual) + } + } + } + } + + private fun read(name: String): ByteBuf { + Js5CompressionTest::class.java.getResourceAsStream(name).use { input -> + return Unpooled.wrappedBuffer(input.readAllBytes()) + } + } +} diff --git a/cache/src/test/resources/dev/openrs2/cache/bzip2-encrypted.dat b/cache/src/test/resources/dev/openrs2/cache/bzip2-encrypted.dat new file mode 100644 index 0000000000000000000000000000000000000000..7d9f4eed925a328feda408f4039fd86316cb430a GIT binary patch literal 55 zcmV-70LcFV0000kg87B~lEdSH(Sfn6M`6VfXhHI1pWPK7fjOyY{#?w^JQ}GBoE-OT N-@nxH(?Hc|Hn8M*96kU5 literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/dev/openrs2/cache/bzip2.dat b/cache/src/test/resources/dev/openrs2/cache/bzip2.dat new file mode 100644 index 0000000000000000000000000000000000000000..0b86ade396698464c818fa90ed94c1624b5fda1d GIT binary patch literal 55 zcmZQ%U|`S#Vs=BvNVVX|_L$^sAg{kc07%YYU}AJ&P+(A!N)BjaT+Q&YTqU@blcV&Q G*#-a$vJ6%L literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/dev/openrs2/cache/gzip-encrypted.dat b/cache/src/test/resources/dev/openrs2/cache/gzip-encrypted.dat new file mode 100644 index 0000000000000000000000000000000000000000..c88e3817f4e334d0f0801f07d67d3fec6d711c10 GIT binary patch literal 34 pcmZQ#U|^8!*{*Ju^pN9ii2uYz#>>C#S3k9*?0Ib>!whzyECARJ4B7wy literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/dev/openrs2/cache/gzip.dat b/cache/src/test/resources/dev/openrs2/cache/gzip.dat new file mode 100644 index 0000000000000000000000000000000000000000..8e2ff3cf82f7d3d422f2ed9c7eadb3c69ff7a645 GIT binary patch literal 34 ncmZQ#U|^60Vs;J&yM+