Add JS5 compression/encryption implementation

Signed-off-by: Graham <gpe@openrs2.dev>
Graham 4 years ago
parent f81f4a81c7
commit 58335ca6d0
  1. 26
      cache/build.gradle.kts
  2. 126
      cache/src/main/java/dev/openrs2/cache/Js5Compression.kt
  3. 51
      cache/src/main/java/dev/openrs2/cache/Js5CompressionType.kt
  4. 231
      cache/src/test/java/dev/openrs2/cache/Js5CompressionTest.kt
  5. BIN
      cache/src/test/resources/dev/openrs2/cache/bzip2-encrypted.dat
  6. BIN
      cache/src/test/resources/dev/openrs2/cache/bzip2.dat
  7. BIN
      cache/src/test/resources/dev/openrs2/cache/gzip-encrypted.dat
  8. BIN
      cache/src/test/resources/dev/openrs2/cache/gzip.dat
  9. BIN
      cache/src/test/resources/dev/openrs2/cache/lzma-encrypted.dat
  10. BIN
      cache/src/test/resources/dev/openrs2/cache/lzma.dat
  11. BIN
      cache/src/test/resources/dev/openrs2/cache/none-encrypted.dat
  12. BIN
      cache/src/test/resources/dev/openrs2/cache/none.dat
  13. 1
      settings.gradle.kts

@ -0,0 +1,26 @@
plugins {
`maven-publish`
kotlin("jvm")
}
dependencies {
implementation(project(":buffer"))
implementation(project(":compress"))
implementation(project(":crypto"))
}
publishing {
publications.create<MavenPublication>("maven") {
from(components["java"])
pom {
packaging = "jar"
name.set("OpenRS2 Cache")
description.set(
"""
A library for reading and writing the RuneScape cache.
""".trimIndent()
)
}
}
}

@ -0,0 +1,126 @@
package dev.openrs2.cache
import dev.openrs2.buffer.use
import dev.openrs2.crypto.XteaKey
import dev.openrs2.crypto.xteaDecrypt
import dev.openrs2.crypto.xteaEncrypt
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufInputStream
import io.netty.buffer.ByteBufOutputStream
object Js5Compression {
fun compress(input: ByteBuf, type: Js5CompressionType, key: XteaKey = XteaKey.ZERO): ByteBuf {
input.alloc().buffer().use { output ->
output.writeByte(type.ordinal)
if (type == Js5CompressionType.NONE) {
val len = input.readableBytes()
output.writeInt(len)
output.writeBytes(input)
if (!key.isZero) {
output.xteaEncrypt(5, len, key)
}
return output.retain()
}
val lenIndex = output.writerIndex()
output.writeZero(4)
output.writeInt(input.readableBytes())
val start = output.writerIndex()
type.createOutputStream(ByteBufOutputStream(output)).use { outputStream ->
ByteBufInputStream(input).use { inputStream ->
inputStream.copyTo(outputStream)
}
}
val len = output.writerIndex() - start
output.setInt(lenIndex, len)
if (!key.isZero) {
output.xteaEncrypt(5, len + 4, key)
}
return output.retain()
}
}
fun compressBest(input: ByteBuf, enableLzma: Boolean = false, key: XteaKey = XteaKey.ZERO): ByteBuf {
var best = compress(input.slice(), Js5CompressionType.NONE, key)
try {
for (type in Js5CompressionType.values()) {
if (type == Js5CompressionType.NONE || (type == Js5CompressionType.LZMA && !enableLzma)) {
continue
}
compress(input.slice(), type, key).use { output ->
if (output.readableBytes() < best.readableBytes()) {
best.release()
best = output.retain()
}
}
}
// consume all of input so this method is a drop-in replacement for compress()
input.skipBytes(input.readableBytes())
return best.retain()
} finally {
best.release()
}
}
fun uncompress(input: ByteBuf, key: XteaKey = XteaKey.ZERO): ByteBuf {
val typeId = input.readUnsignedByte().toInt()
val type = Js5CompressionType.fromOrdinal(typeId)
require(type != null) {
"Invalid compression type: $typeId"
}
val len = input.readInt()
require(len >= 0) {
"Length is negative: $len"
}
if (type == Js5CompressionType.NONE) {
input.readBytes(len).use { output ->
if (!key.isZero) {
output.xteaDecrypt(0, len, key)
}
return output.retain()
}
}
decrypt(input, len + 4, key).use { plaintext ->
val uncompressedLen = plaintext.readInt()
require(uncompressedLen >= 0) {
"Uncompressed length is negative: $uncompressedLen"
}
plaintext.alloc().buffer(uncompressedLen, uncompressedLen).use { output ->
type.createInputStream(ByteBufInputStream(plaintext, len), uncompressedLen).use { inputStream ->
ByteBufOutputStream(output).use { outputStream ->
inputStream.copyTo(outputStream)
}
}
return output.retain()
}
}
}
private fun decrypt(buf: ByteBuf, len: Int, key: XteaKey): ByteBuf {
if (key.isZero) {
return buf.readRetainedSlice(len)
}
buf.readBytes(len).use { output ->
output.xteaDecrypt(0, len, key)
return output.retain()
}
}
}

@ -0,0 +1,51 @@
package dev.openrs2.cache
import dev.openrs2.compress.bzip2.Bzip2
import dev.openrs2.compress.gzip.Gzip
import dev.openrs2.compress.lzma.Lzma
import java.io.InputStream
import java.io.OutputStream
import java.util.zip.Deflater
enum class Js5CompressionType {
NONE,
BZIP2,
GZIP,
LZMA;
fun createInputStream(input: InputStream, length: Int): InputStream {
return when (this) {
NONE -> input
BZIP2 -> Bzip2.createHeaderlessInputStream(input)
GZIP -> Gzip.createHeaderlessInputStream(input)
LZMA -> Lzma.createHeaderlessInputStream(input, length.toLong())
}
}
fun createOutputStream(output: OutputStream): OutputStream {
return when (this) {
NONE -> output
BZIP2 -> Bzip2.createHeaderlessOutputStream(output)
GZIP -> Gzip.createHeaderlessOutputStream(output, Deflater.BEST_COMPRESSION)
/*
* LZMA at -9 has significantly higher CPU/memory requirements for
* both compression _and_ decompression, so we use the default of
* -6. Using a higher level for the typical file size in the
* RuneScape cache probably provides insignificant returns, as
* described in the LZMA documentation.
*/
LZMA -> Lzma.createHeaderlessOutputStream(output, Lzma.DEFAULT_COMPRESSION)
}
}
companion object {
fun fromOrdinal(ordinal: Int): Js5CompressionType? {
val values = values()
return if (ordinal >= 0 && ordinal < values.size) {
values[ordinal]
} else {
null
}
}
}
}

@ -0,0 +1,231 @@
package dev.openrs2.cache
import dev.openrs2.buffer.use
import dev.openrs2.crypto.XteaKey
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
object Js5CompressionTest {
private val KEY = XteaKey(intArrayOf(0x00112233, 0x44556677, 0x8899AABB.toInt(), 0xCCDDEEFF.toInt()))
@Test
fun testCompressNone() {
read("none.dat").use { expected ->
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { input ->
Js5Compression.compress(input, Js5CompressionType.NONE).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressNone() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("none.dat").use { input ->
Js5Compression.uncompress(input).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressGzip() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP).use { compressed ->
Js5Compression.uncompress(compressed).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressGzip() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("gzip.dat").use { input ->
Js5Compression.uncompress(input).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressBzip2() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2).use { compressed ->
Js5Compression.uncompress(compressed).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressBzip2() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("bzip2.dat").use { input ->
Js5Compression.uncompress(input).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressLzma() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA).use { compressed ->
Js5Compression.uncompress(compressed).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressLzma() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("lzma.dat").use { input ->
Js5Compression.uncompress(input).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressNoneEncrypted() {
read("none-encrypted.dat").use { expected ->
Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { input ->
Js5Compression.compress(input, Js5CompressionType.NONE, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressNoneEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".repeat(3).toByteArray()).use { expected ->
read("none-encrypted.dat").use { input ->
Js5Compression.uncompress(input, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressGzipEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.GZIP, KEY).use { compressed ->
Js5Compression.uncompress(compressed, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressGzipEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("gzip-encrypted.dat").use { input ->
Js5Compression.uncompress(input, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressBzip2Encrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.BZIP2, KEY).use { compressed ->
Js5Compression.uncompress(compressed, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressBzip2Encrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("bzip2-encrypted.dat").use { input ->
Js5Compression.uncompress(input, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressLzmaEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
Js5Compression.compress(expected.slice(), Js5CompressionType.LZMA, KEY).use { compressed ->
Js5Compression.uncompress(compressed, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testUncompressLzmaEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".toByteArray()).use { expected ->
read("lzma-encrypted.dat").use { input ->
Js5Compression.uncompress(input, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressBest() {
Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected ->
val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed ->
compressed.readableBytes()
}
Js5Compression.compressBest(expected.slice()).use { compressed ->
assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt())
assert(compressed.readableBytes() < noneLen)
Js5Compression.uncompress(compressed).use { actual ->
assertEquals(expected, actual)
}
}
}
}
@Test
fun testCompressBestEncrypted() {
Unpooled.wrappedBuffer("OpenRS2".repeat(100).toByteArray()).use { expected ->
val noneLen = Js5Compression.compress(expected.slice(), Js5CompressionType.NONE).use { compressed ->
compressed.readableBytes()
}
Js5Compression.compressBest(expected.slice(), key = KEY).use { compressed ->
assertNotEquals(Js5CompressionType.NONE.ordinal, compressed.getUnsignedByte(0).toInt())
assert(compressed.readableBytes() < noneLen)
Js5Compression.uncompress(compressed, KEY).use { actual ->
assertEquals(expected, actual)
}
}
}
}
private fun read(name: String): ByteBuf {
Js5CompressionTest::class.java.getResourceAsStream(name).use { input ->
return Unpooled.wrappedBuffer(input.readAllBytes())
}
}
}

@ -5,6 +5,7 @@ include(
"asm", "asm",
"buffer", "buffer",
"bundler", "bundler",
"cache",
"compress", "compress",
"compress-cli", "compress-cli",
"conf", "conf",

Loading…
Cancel
Save