From 26c6ad21e3bc2a39ee01dc5a9f692661ba88170f Mon Sep 17 00:00:00 2001 From: Graham Date: Tue, 30 Nov 2021 22:44:17 +0000 Subject: [PATCH] Add class for reading and writing .jag archives Signed-off-by: Graham --- .../kotlin/org/openrs2/cache/JagArchive.kt | 293 ++++++++++++++++++ .../org/openrs2/cache/JagArchiveTest.kt | 198 ++++++++++++ .../openrs2/cache/jag/duplicate-entries.jag | Bin 0 -> 113 bytes .../cache/jag/empty-compressed-archive.jag | Bin 0 -> 39 bytes .../cache/jag/empty-compressed-entries.jag | Bin 0 -> 8 bytes .../cache/jag/multiple-compressed-archive.jag | Bin 0 -> 88 bytes .../cache/jag/multiple-compressed-entries.jag | Bin 0 -> 113 bytes .../cache/jag/single-compressed-archive.jag | Bin 0 -> 68 bytes .../cache/jag/single-compressed-entries.jag | Bin 0 -> 64 bytes 9 files changed, 491 insertions(+) create mode 100644 cache/src/main/kotlin/org/openrs2/cache/JagArchive.kt create mode 100644 cache/src/test/kotlin/org/openrs2/cache/JagArchiveTest.kt create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/duplicate-entries.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-archive.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-entries.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-archive.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-entries.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/single-compressed-archive.jag create mode 100644 cache/src/test/resources/org/openrs2/cache/jag/single-compressed-entries.jag diff --git a/cache/src/main/kotlin/org/openrs2/cache/JagArchive.kt b/cache/src/main/kotlin/org/openrs2/cache/JagArchive.kt new file mode 100644 index 00000000..9159beef --- /dev/null +++ b/cache/src/main/kotlin/org/openrs2/cache/JagArchive.kt @@ -0,0 +1,293 @@ +package org.openrs2.cache + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.ByteBufInputStream +import io.netty.buffer.ByteBufOutputStream +import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap +import org.openrs2.buffer.use +import org.openrs2.compress.bzip2.Bzip2 +import org.openrs2.util.jagHashCode +import java.io.Closeable +import java.io.FileNotFoundException + +/** + * An interface for reading and writing `.jag` archives, which are used by + * RuneScape Classic and early versions of RuneScape 2. + * + * Unlike the client, this implementation is case-sensitive. Entry names should + * therefore be supplied in uppercase for compatibility. + * + * This class is not thread safe. + */ +public class JagArchive : Closeable { + private val entries = Int2ObjectLinkedOpenHashMap() + + /** + * The number of entries in the archive. + */ + public val size: Int + get() = entries.size + + /** + * Lists all entries in the archive. + * @return a sorted list of entry name hashes. + */ + public fun list(): Iterator { + return entries.keys.iterator() + } + + /** + * Checks whether an entry exists. + * @param name the entry's name. + * @return `true` if so, `false` otherwise. + */ + public fun exists(name: String): Boolean { + return existsNamed(name.jagHashCode()) + } + + /** + * Checks whether an entry exists. + * @param nameHash the entry's name hash. + * @return `true` if so, `false` otherwise. + */ + public fun existsNamed(nameHash: Int): Boolean { + return entries.containsKey(nameHash) + } + + /** + * Reads an entry. + * + * This method allocates and returns a new [ByteBuf]. It is the caller's + * responsibility to release the [ByteBuf]. + * @param name the entry's name. + * @return the contents of the entry. + * @throws FileNotFoundException if the entry does not exist. + */ + public fun read(name: String): ByteBuf { + return readNamed(name.jagHashCode()) + } + + /** + * Reads an entry. + * + * This method allocates and returns a new [ByteBuf]. It is the caller's + * responsibility to release the [ByteBuf]. + * @param nameHash the entry's name hash. + * @return the contents of the entry. + * @throws FileNotFoundException if the entry does not exist. + */ + public fun readNamed(nameHash: Int): ByteBuf { + val buf = entries[nameHash] ?: throw FileNotFoundException() + return buf.retainedSlice() + } + + /** + * Writes an entry. If an entry with the same name hash already exists, it + * is overwritten. + * @param name the entry's name. + * @param buf the new contents of the entry. + */ + public fun write(name: String, buf: ByteBuf) { + writeNamed(name.jagHashCode(), buf) + } + + /** + * Writes an entry. If an entry with the same name hash already exists, it + * is overwritten. + * @param nameHash the entry's name hash. + * @param buf the new contents of the entry. + */ + public fun writeNamed(nameHash: Int, buf: ByteBuf) { + entries.put(nameHash, buf.copy().asReadOnly())?.release() + } + + /** + * Deletes an entry. Does nothing if the entry does not exist. + * @param name the entry's name. + */ + public fun remove(name: String) { + removeNamed(name.jagHashCode()) + } + + /** + * Deletes an entry. Does nothing if the entry does not exist. + * @param nameHash the entry's name hash. + */ + public fun removeNamed(nameHash: Int) { + entries.remove(nameHash)?.release() + } + + /** + * Packs a `.jag` archive into a compressed [ByteBuf] using the given + * compression method. + * + * This method allocates and returns a new [ByteBuf]. It is the caller's + * responsibility to release the [ByteBuf]. + * @param compressedArchive `true` if the archive should be compressed as a + * whole, `false` if each entry should be compressed individually. + * @param alloc the allocator. + * @return the compressed archive. + */ + public fun pack(compressedArchive: Boolean, alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { + alloc.buffer().use { output -> + alloc.buffer().use { uncompressedArchiveBuf -> + uncompressedArchiveBuf.writeShort(size) + var index = uncompressedArchiveBuf.writerIndex() + size * 10 + + for ((nameHash, uncompressedEntryBuf) in entries) { + uncompressedArchiveBuf.writeInt(nameHash) + uncompressedArchiveBuf.writeMedium(uncompressedEntryBuf.readableBytes()) + + compress(uncompressedEntryBuf.slice(), !compressedArchive).use { entryBuf -> + val entryLen = entryBuf.readableBytes() + uncompressedArchiveBuf.writeMedium(entryLen) + + uncompressedArchiveBuf.setBytes(index, entryBuf) + index += entryLen + } + } + + uncompressedArchiveBuf.writerIndex(index) + + val uncompressedLen = uncompressedArchiveBuf.readableBytes() + output.writeMedium(uncompressedLen) + + compress(uncompressedArchiveBuf, compressedArchive).use { archiveBuf -> + val len = archiveBuf.readableBytes() + if (compressedArchive && uncompressedLen == len) { + /* + * This is a bit of an odd corner case. If whole + * archive compression is enabled and the lengths are + * equal we have to use individual entry compression + * instead, as the lengths being equal signals to the + * client that this mode should be used. + * + * If anyone finds a suitable test case for this case, + * I'd love to see it! + */ + return pack(false, alloc) + } + + output.writeMedium(len) + output.writeBytes(archiveBuf) + } + + return output.retain() + } + } + } + + /** + * Packs a `.jag` archive into a compressed [ByteBuf]. The best compression + * method for minimising the size of the compressed archive is + * automatically selected. Note that this does not necessarily correspond + * to the minimimal amount of RAM usage at runtime. + * + * This method allocates and returns a new [ByteBuf]. It is the caller's + * responsibility to release the [ByteBuf]. + * @param alloc the allocator. + * @return the compressed archive. + */ + public fun packBest(alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { + pack(true, alloc).use { compressedArchive -> + pack(false, alloc).use { compressedEntries -> + // If equal, pick the archive that only requires one + // decompression instead of several to save CPU time. + return if (compressedArchive.readableBytes() <= compressedEntries.readableBytes()) { + compressedArchive.retain() + } else { + compressedEntries.retain() + } + } + } + } + + override fun close() { + entries.values.forEach(ByteBuf::release) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as JagArchive + + if (entries != other.entries) return false + + return true + } + + override fun hashCode(): Int { + return entries.hashCode() + } + + public companion object { + /** + * Unpacks a [ByteBuf] containing a compressed `.jag` archive. + * @param buf the compressed archive. + * @return the unpacked archive. + */ + public fun unpack(buf: ByteBuf): JagArchive { + val archive = JagArchive() + + val uncompressedLen = buf.readUnsignedMedium() + val len = buf.readUnsignedMedium() + + val compressedArchive = len != uncompressedLen + + uncompress(buf, compressedArchive, len, uncompressedLen).use { archiveBuf -> + val size = archiveBuf.readUnsignedShort() + var index = archiveBuf.readerIndex() + size * 10 + + for (id in 0 until size) { + val nameHash = archiveBuf.readInt() + val uncompressedEntryLen = archiveBuf.readUnsignedMedium() + val entryLen = archiveBuf.readUnsignedMedium() + + val entry = archiveBuf.slice(index, entryLen) + + uncompress(entry, !compressedArchive, entryLen, uncompressedEntryLen).use { entryBuf -> + if (!archive.existsNamed(nameHash)) { + // Store the first entry if there is a collision, + // for compatibility with the client. + archive.writeNamed(nameHash, entryBuf) + } + } + + index += entryLen + } + } + + return archive + } + + private fun compress(input: ByteBuf, compressed: Boolean): ByteBuf { + return if (compressed) { + input.alloc().buffer().use { output -> + Bzip2.createHeaderlessOutputStream(ByteBufOutputStream(output)).use { stream -> + input.readBytes(stream, input.readableBytes()) + } + + output.retain() + } + } else { + input.readRetainedSlice(input.readableBytes()) + } + } + + private fun uncompress(buf: ByteBuf, compressed: Boolean, compressedLen: Int, uncompressedLen: Int): ByteBuf { + return if (compressed) { + buf.alloc().buffer(uncompressedLen, uncompressedLen).use { output -> + Bzip2.createHeaderlessInputStream(ByteBufInputStream(buf.readSlice(compressedLen))).use { stream -> + output.writeBytes(stream, uncompressedLen) + } + + output.retain() + } + } else { + buf.readRetainedSlice(compressedLen) + } + } + } +} diff --git a/cache/src/test/kotlin/org/openrs2/cache/JagArchiveTest.kt b/cache/src/test/kotlin/org/openrs2/cache/JagArchiveTest.kt new file mode 100644 index 00000000..71740ce0 --- /dev/null +++ b/cache/src/test/kotlin/org/openrs2/cache/JagArchiveTest.kt @@ -0,0 +1,198 @@ +package org.openrs2.cache + +import io.netty.buffer.Unpooled +import org.openrs2.buffer.copiedBuffer +import org.openrs2.buffer.use +import org.openrs2.util.jagHashCode +import java.nio.file.Files +import java.nio.file.Path +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class JagArchiveTest { + @Test + fun testEmpty() { + JagArchive().use { archive -> + assertEquals(0, archive.size) + assertEquals(emptyList(), archive.list().asSequence().toList()) + + packTest("empty-compressed-archive.jag", archive, true) + packTest("empty-compressed-entries.jag", archive, false) + packBestTest("empty-compressed-entries.jag", archive) + + unpackTest("empty-compressed-archive.jag", archive) + unpackTest("empty-compressed-entries.jag", archive) + } + } + + @Test + fun testSingleEntry() { + JagArchive().use { archive -> + assertEquals(0, archive.size) + assertEquals(emptyList(), archive.list().asSequence().toList()) + assertFalse(archive.exists("TEST.TXT")) + assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) + + copiedBuffer("OpenRS2").use { expected -> + archive.write("TEST.TXT", expected) + + archive.read("TEST.TXT").use { actual -> + assertEquals(expected, actual) + } + + archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> + assertEquals(expected, actual) + } + } + + assertEquals(1, archive.size) + assertEquals(listOf("TEST.TXT".jagHashCode()), archive.list().asSequence().toList()) + assertTrue(archive.exists("TEST.TXT")) + assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) + assertFalse(archive.exists("HELLO.TXT")) + assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) + + packTest("single-compressed-archive.jag", archive, true) + packTest("single-compressed-entries.jag", archive, false) + packBestTest("single-compressed-entries.jag", archive) + + unpackTest("single-compressed-archive.jag", archive) + unpackTest("single-compressed-entries.jag", archive) + } + } + + @Test + fun testMultipleEntries() { + JagArchive().use { archive -> + assertEquals(0, archive.size) + assertEquals(emptyList(), archive.list().asSequence().toList()) + assertFalse(archive.exists("TEST.TXT")) + assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) + assertFalse(archive.exists("HELLO.TXT")) + assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) + + copiedBuffer("OpenRS2").use { expected -> + archive.write("TEST.TXT", expected) + + archive.read("TEST.TXT").use { actual -> + assertEquals(expected, actual) + } + + archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> + assertEquals(expected, actual) + } + } + + copiedBuffer("Hello").use { expected -> + archive.write("HELLO.TXT", expected) + + archive.read("HELLO.TXT").use { actual -> + assertEquals(expected, actual) + } + + archive.readNamed("HELLO.TXT".jagHashCode()).use { actual -> + assertEquals(expected, actual) + } + } + + assertEquals(2, archive.size) + assertEquals( + listOf( + "TEST.TXT".jagHashCode(), + "HELLO.TXT".jagHashCode() + ), archive.list().asSequence().toList() + ) + assertTrue(archive.exists("TEST.TXT")) + assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) + assertTrue(archive.exists("HELLO.TXT")) + assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) + assertFalse(archive.exists("OTHER.TXT")) + assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) + + packTest("multiple-compressed-archive.jag", archive, true) + packTest("multiple-compressed-entries.jag", archive, false) + packBestTest("multiple-compressed-archive.jag", archive) + + unpackTest("multiple-compressed-archive.jag", archive) + unpackTest("multiple-compressed-entries.jag", archive) + + archive.remove("TEST.TXT") + + assertEquals(1, archive.size) + assertEquals(listOf("HELLO.TXT".jagHashCode()), archive.list().asSequence().toList()) + assertFalse(archive.exists("TEST.TXT")) + assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) + assertTrue(archive.exists("HELLO.TXT")) + assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) + assertFalse(archive.exists("OTHER.TXT")) + assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) + + archive.remove("TEST.TXT") + archive.removeNamed("HELLO.TXT".jagHashCode()) // check remove a non-existent entry works + + assertEquals(0, archive.size) + assertEquals(emptyList(), archive.list().asSequence().toList()) + assertFalse(archive.exists("TEST.TXT")) + assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) + assertFalse(archive.exists("HELLO.TXT")) + assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) + assertFalse(archive.exists("OTHER.TXT")) + assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) + + archive.removeNamed("HELLO.TXT".jagHashCode()) + + archive.remove("OTHER.TXT") // check removing an entry that never existed works + archive.removeNamed("OTHER.TXT".jagHashCode()) + + assertEquals(0, archive.size) + assertEquals(emptyList(), archive.list().asSequence().toList()) + assertFalse(archive.exists("TEST.TXT")) + assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) + assertFalse(archive.exists("HELLO.TXT")) + assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) + assertFalse(archive.exists("OTHER.TXT")) + assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) + } + } + + @Test + fun testDuplicateEntries() { + JagArchive().use { archive -> + copiedBuffer("OpenRS2").use { buf -> + archive.write("TEST.TXT", buf) + } + + unpackTest("duplicate-entries.jag", archive) + } + } + + private fun packTest(name: String, archive: JagArchive, compressedArchive: Boolean) { + Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> + archive.pack(compressedArchive).use { actual -> + assertEquals(expected, actual) + } + } + } + + private fun packBestTest(name: String, archive: JagArchive) { + Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> + archive.packBest().use { actual -> + assertEquals(expected, actual) + } + } + } + + private fun unpackTest(name: String, expected: JagArchive) { + Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { buf -> + JagArchive.unpack(buf).use { actual -> + assertEquals(expected, actual) + } + } + } + + private companion object { + private val ROOT = Path.of(JagArchiveTest::class.java.getResource("jag").toURI()) + } +} diff --git a/cache/src/test/resources/org/openrs2/cache/jag/duplicate-entries.jag b/cache/src/test/resources/org/openrs2/cache/jag/duplicate-entries.jag new file mode 100644 index 0000000000000000000000000000000000000000..2b23b63e47291d83940cb386d47fcb7cb0e5fdff GIT binary patch literal 113 zcmZQz$YuZ{rpu@PUt(ZjXJF8Suvi%w)D0aY)q*42W0HYN82TFofaDAYCPoJa1qMSN zMqa}eR~ssJPtrWp(qi>kIs&3vDkNnRP#sXa0~5;v27|LElFK@09=$WsPeg@ns-#K- E0Pr;+O#lD@ literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-archive.jag b/cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-archive.jag new file mode 100644 index 0000000000000000000000000000000000000000..2e5f0b0167fad7c09e9a04ee98bc76318960832e GIT binary patch literal 39 tcmZQzU}9iUG<1wq3y%EnF}sO@fx!U?6&MV9loQKUf?GNHzjW?U005!Z3FQC) literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-entries.jag b/cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-entries.jag new file mode 100644 index 0000000000000000000000000000000000000000..3017d3b83e8bc43df51e04d3a8417a716a3d1c6c GIT binary patch literal 8 McmZQzU}69v000^Q1poj5 literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-archive.jag b/cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-archive.jag new file mode 100644 index 0000000000000000000000000000000000000000..9f24673890c9586d4b7d80de0e47dd9ab9e052a5 GIT binary patch literal 88 zcmV-e0H^-|03rYYQZYeUCR14-tYrxR00M8P(trd25C8x`m;eF=zyJUN01y}eKmY&; upa3yOjDe#}jS6T0X{V{Ci4ioL+~qu1n=!yd0!z+iCJL~>c@%%gWE`iZEp JO_fw>005s7A8Y^s literal 0 HcmV?d00001 diff --git a/cache/src/test/resources/org/openrs2/cache/jag/single-compressed-archive.jag b/cache/src/test/resources/org/openrs2/cache/jag/single-compressed-archive.jag new file mode 100644 index 0000000000000000000000000000000000000000..91ffb2023608f2687d20b54477ebb55bb9b27707 GIT binary patch literal 68 zcmZQz5N2SoGjxnp3y$pl`Xz>efziM1L4yDT1H%jkCPoJa1_22mdjW$X0|STpqKq#h XGFl%0JI$xh(tFZXuuxI@^@|+=PvUt(ZjXJF7Xbc|FBj%<%f&Sqd>=x-1Jk~0{X7#$cC7z}wBc@0-w RZK&8iN%K%ki`8T42mt8k5kdd} literal 0 HcmV?d00001