Add class for reading and writing .jag archives

Signed-off-by: Graham <gpe@openrs2.org>
Graham 3 years ago
parent 3734d4709b
commit 26c6ad21e3
  1. 293
      cache/src/main/kotlin/org/openrs2/cache/JagArchive.kt
  2. 198
      cache/src/test/kotlin/org/openrs2/cache/JagArchiveTest.kt
  3. BIN
      cache/src/test/resources/org/openrs2/cache/jag/duplicate-entries.jag
  4. BIN
      cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-archive.jag
  5. BIN
      cache/src/test/resources/org/openrs2/cache/jag/empty-compressed-entries.jag
  6. BIN
      cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-archive.jag
  7. BIN
      cache/src/test/resources/org/openrs2/cache/jag/multiple-compressed-entries.jag
  8. BIN
      cache/src/test/resources/org/openrs2/cache/jag/single-compressed-archive.jag
  9. BIN
      cache/src/test/resources/org/openrs2/cache/jag/single-compressed-entries.jag

@ -0,0 +1,293 @@
package org.openrs2.cache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.ByteBufInputStream
import io.netty.buffer.ByteBufOutputStream
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap
import org.openrs2.buffer.use
import org.openrs2.compress.bzip2.Bzip2
import org.openrs2.util.jagHashCode
import java.io.Closeable
import java.io.FileNotFoundException
/**
* An interface for reading and writing `.jag` archives, which are used by
* RuneScape Classic and early versions of RuneScape 2.
*
* Unlike the client, this implementation is case-sensitive. Entry names should
* therefore be supplied in uppercase for compatibility.
*
* This class is not thread safe.
*/
public class JagArchive : Closeable {
private val entries = Int2ObjectLinkedOpenHashMap<ByteBuf>()
/**
* The number of entries in the archive.
*/
public val size: Int
get() = entries.size
/**
* Lists all entries in the archive.
* @return a sorted list of entry name hashes.
*/
public fun list(): Iterator<Int> {
return entries.keys.iterator()
}
/**
* Checks whether an entry exists.
* @param name the entry's name.
* @return `true` if so, `false` otherwise.
*/
public fun exists(name: String): Boolean {
return existsNamed(name.jagHashCode())
}
/**
* Checks whether an entry exists.
* @param nameHash the entry's name hash.
* @return `true` if so, `false` otherwise.
*/
public fun existsNamed(nameHash: Int): Boolean {
return entries.containsKey(nameHash)
}
/**
* Reads an entry.
*
* This method allocates and returns a new [ByteBuf]. It is the caller's
* responsibility to release the [ByteBuf].
* @param name the entry's name.
* @return the contents of the entry.
* @throws FileNotFoundException if the entry does not exist.
*/
public fun read(name: String): ByteBuf {
return readNamed(name.jagHashCode())
}
/**
* Reads an entry.
*
* This method allocates and returns a new [ByteBuf]. It is the caller's
* responsibility to release the [ByteBuf].
* @param nameHash the entry's name hash.
* @return the contents of the entry.
* @throws FileNotFoundException if the entry does not exist.
*/
public fun readNamed(nameHash: Int): ByteBuf {
val buf = entries[nameHash] ?: throw FileNotFoundException()
return buf.retainedSlice()
}
/**
* Writes an entry. If an entry with the same name hash already exists, it
* is overwritten.
* @param name the entry's name.
* @param buf the new contents of the entry.
*/
public fun write(name: String, buf: ByteBuf) {
writeNamed(name.jagHashCode(), buf)
}
/**
* Writes an entry. If an entry with the same name hash already exists, it
* is overwritten.
* @param nameHash the entry's name hash.
* @param buf the new contents of the entry.
*/
public fun writeNamed(nameHash: Int, buf: ByteBuf) {
entries.put(nameHash, buf.copy().asReadOnly())?.release()
}
/**
* Deletes an entry. Does nothing if the entry does not exist.
* @param name the entry's name.
*/
public fun remove(name: String) {
removeNamed(name.jagHashCode())
}
/**
* Deletes an entry. Does nothing if the entry does not exist.
* @param nameHash the entry's name hash.
*/
public fun removeNamed(nameHash: Int) {
entries.remove(nameHash)?.release()
}
/**
* Packs a `.jag` archive into a compressed [ByteBuf] using the given
* compression method.
*
* This method allocates and returns a new [ByteBuf]. It is the caller's
* responsibility to release the [ByteBuf].
* @param compressedArchive `true` if the archive should be compressed as a
* whole, `false` if each entry should be compressed individually.
* @param alloc the allocator.
* @return the compressed archive.
*/
public fun pack(compressedArchive: Boolean, alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf {
alloc.buffer().use { output ->
alloc.buffer().use { uncompressedArchiveBuf ->
uncompressedArchiveBuf.writeShort(size)
var index = uncompressedArchiveBuf.writerIndex() + size * 10
for ((nameHash, uncompressedEntryBuf) in entries) {
uncompressedArchiveBuf.writeInt(nameHash)
uncompressedArchiveBuf.writeMedium(uncompressedEntryBuf.readableBytes())
compress(uncompressedEntryBuf.slice(), !compressedArchive).use { entryBuf ->
val entryLen = entryBuf.readableBytes()
uncompressedArchiveBuf.writeMedium(entryLen)
uncompressedArchiveBuf.setBytes(index, entryBuf)
index += entryLen
}
}
uncompressedArchiveBuf.writerIndex(index)
val uncompressedLen = uncompressedArchiveBuf.readableBytes()
output.writeMedium(uncompressedLen)
compress(uncompressedArchiveBuf, compressedArchive).use { archiveBuf ->
val len = archiveBuf.readableBytes()
if (compressedArchive && uncompressedLen == len) {
/*
* This is a bit of an odd corner case. If whole
* archive compression is enabled and the lengths are
* equal we have to use individual entry compression
* instead, as the lengths being equal signals to the
* client that this mode should be used.
*
* If anyone finds a suitable test case for this case,
* I'd love to see it!
*/
return pack(false, alloc)
}
output.writeMedium(len)
output.writeBytes(archiveBuf)
}
return output.retain()
}
}
}
/**
* Packs a `.jag` archive into a compressed [ByteBuf]. The best compression
* method for minimising the size of the compressed archive is
* automatically selected. Note that this does not necessarily correspond
* to the minimimal amount of RAM usage at runtime.
*
* This method allocates and returns a new [ByteBuf]. It is the caller's
* responsibility to release the [ByteBuf].
* @param alloc the allocator.
* @return the compressed archive.
*/
public fun packBest(alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf {
pack(true, alloc).use { compressedArchive ->
pack(false, alloc).use { compressedEntries ->
// If equal, pick the archive that only requires one
// decompression instead of several to save CPU time.
return if (compressedArchive.readableBytes() <= compressedEntries.readableBytes()) {
compressedArchive.retain()
} else {
compressedEntries.retain()
}
}
}
}
override fun close() {
entries.values.forEach(ByteBuf::release)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as JagArchive
if (entries != other.entries) return false
return true
}
override fun hashCode(): Int {
return entries.hashCode()
}
public companion object {
/**
* Unpacks a [ByteBuf] containing a compressed `.jag` archive.
* @param buf the compressed archive.
* @return the unpacked archive.
*/
public fun unpack(buf: ByteBuf): JagArchive {
val archive = JagArchive()
val uncompressedLen = buf.readUnsignedMedium()
val len = buf.readUnsignedMedium()
val compressedArchive = len != uncompressedLen
uncompress(buf, compressedArchive, len, uncompressedLen).use { archiveBuf ->
val size = archiveBuf.readUnsignedShort()
var index = archiveBuf.readerIndex() + size * 10
for (id in 0 until size) {
val nameHash = archiveBuf.readInt()
val uncompressedEntryLen = archiveBuf.readUnsignedMedium()
val entryLen = archiveBuf.readUnsignedMedium()
val entry = archiveBuf.slice(index, entryLen)
uncompress(entry, !compressedArchive, entryLen, uncompressedEntryLen).use { entryBuf ->
if (!archive.existsNamed(nameHash)) {
// Store the first entry if there is a collision,
// for compatibility with the client.
archive.writeNamed(nameHash, entryBuf)
}
}
index += entryLen
}
}
return archive
}
private fun compress(input: ByteBuf, compressed: Boolean): ByteBuf {
return if (compressed) {
input.alloc().buffer().use { output ->
Bzip2.createHeaderlessOutputStream(ByteBufOutputStream(output)).use { stream ->
input.readBytes(stream, input.readableBytes())
}
output.retain()
}
} else {
input.readRetainedSlice(input.readableBytes())
}
}
private fun uncompress(buf: ByteBuf, compressed: Boolean, compressedLen: Int, uncompressedLen: Int): ByteBuf {
return if (compressed) {
buf.alloc().buffer(uncompressedLen, uncompressedLen).use { output ->
Bzip2.createHeaderlessInputStream(ByteBufInputStream(buf.readSlice(compressedLen))).use { stream ->
output.writeBytes(stream, uncompressedLen)
}
output.retain()
}
} else {
buf.readRetainedSlice(compressedLen)
}
}
}
}

@ -0,0 +1,198 @@
package org.openrs2.cache
import io.netty.buffer.Unpooled
import org.openrs2.buffer.copiedBuffer
import org.openrs2.buffer.use
import org.openrs2.util.jagHashCode
import java.nio.file.Files
import java.nio.file.Path
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class JagArchiveTest {
@Test
fun testEmpty() {
JagArchive().use { archive ->
assertEquals(0, archive.size)
assertEquals(emptyList(), archive.list().asSequence().toList())
packTest("empty-compressed-archive.jag", archive, true)
packTest("empty-compressed-entries.jag", archive, false)
packBestTest("empty-compressed-entries.jag", archive)
unpackTest("empty-compressed-archive.jag", archive)
unpackTest("empty-compressed-entries.jag", archive)
}
}
@Test
fun testSingleEntry() {
JagArchive().use { archive ->
assertEquals(0, archive.size)
assertEquals(emptyList(), archive.list().asSequence().toList())
assertFalse(archive.exists("TEST.TXT"))
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode()))
copiedBuffer("OpenRS2").use { expected ->
archive.write("TEST.TXT", expected)
archive.read("TEST.TXT").use { actual ->
assertEquals(expected, actual)
}
archive.readNamed("TEST.TXT".jagHashCode()).use { actual ->
assertEquals(expected, actual)
}
}
assertEquals(1, archive.size)
assertEquals(listOf("TEST.TXT".jagHashCode()), archive.list().asSequence().toList())
assertTrue(archive.exists("TEST.TXT"))
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode()))
assertFalse(archive.exists("HELLO.TXT"))
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode()))
packTest("single-compressed-archive.jag", archive, true)
packTest("single-compressed-entries.jag", archive, false)
packBestTest("single-compressed-entries.jag", archive)
unpackTest("single-compressed-archive.jag", archive)
unpackTest("single-compressed-entries.jag", archive)
}
}
@Test
fun testMultipleEntries() {
JagArchive().use { archive ->
assertEquals(0, archive.size)
assertEquals(emptyList(), archive.list().asSequence().toList())
assertFalse(archive.exists("TEST.TXT"))
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode()))
assertFalse(archive.exists("HELLO.TXT"))
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode()))
copiedBuffer("OpenRS2").use { expected ->
archive.write("TEST.TXT", expected)
archive.read("TEST.TXT").use { actual ->
assertEquals(expected, actual)
}
archive.readNamed("TEST.TXT".jagHashCode()).use { actual ->
assertEquals(expected, actual)
}
}
copiedBuffer("Hello").use { expected ->
archive.write("HELLO.TXT", expected)
archive.read("HELLO.TXT").use { actual ->
assertEquals(expected, actual)
}
archive.readNamed("HELLO.TXT".jagHashCode()).use { actual ->
assertEquals(expected, actual)
}
}
assertEquals(2, archive.size)
assertEquals(
listOf(
"TEST.TXT".jagHashCode(),
"HELLO.TXT".jagHashCode()
), archive.list().asSequence().toList()
)
assertTrue(archive.exists("TEST.TXT"))
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode()))
assertTrue(archive.exists("HELLO.TXT"))
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode()))
assertFalse(archive.exists("OTHER.TXT"))
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode()))
packTest("multiple-compressed-archive.jag", archive, true)
packTest("multiple-compressed-entries.jag", archive, false)
packBestTest("multiple-compressed-archive.jag", archive)
unpackTest("multiple-compressed-archive.jag", archive)
unpackTest("multiple-compressed-entries.jag", archive)
archive.remove("TEST.TXT")
assertEquals(1, archive.size)
assertEquals(listOf("HELLO.TXT".jagHashCode()), archive.list().asSequence().toList())
assertFalse(archive.exists("TEST.TXT"))
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode()))
assertTrue(archive.exists("HELLO.TXT"))
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode()))
assertFalse(archive.exists("OTHER.TXT"))
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode()))
archive.remove("TEST.TXT")
archive.removeNamed("HELLO.TXT".jagHashCode()) // check remove a non-existent entry works
assertEquals(0, archive.size)
assertEquals(emptyList(), archive.list().asSequence().toList())
assertFalse(archive.exists("TEST.TXT"))
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode()))
assertFalse(archive.exists("HELLO.TXT"))
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode()))
assertFalse(archive.exists("OTHER.TXT"))
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode()))
archive.removeNamed("HELLO.TXT".jagHashCode())
archive.remove("OTHER.TXT") // check removing an entry that never existed works
archive.removeNamed("OTHER.TXT".jagHashCode())
assertEquals(0, archive.size)
assertEquals(emptyList(), archive.list().asSequence().toList())
assertFalse(archive.exists("TEST.TXT"))
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode()))
assertFalse(archive.exists("HELLO.TXT"))
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode()))
assertFalse(archive.exists("OTHER.TXT"))
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode()))
}
}
@Test
fun testDuplicateEntries() {
JagArchive().use { archive ->
copiedBuffer("OpenRS2").use { buf ->
archive.write("TEST.TXT", buf)
}
unpackTest("duplicate-entries.jag", archive)
}
}
private fun packTest(name: String, archive: JagArchive, compressedArchive: Boolean) {
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected ->
archive.pack(compressedArchive).use { actual ->
assertEquals(expected, actual)
}
}
}
private fun packBestTest(name: String, archive: JagArchive) {
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected ->
archive.packBest().use { actual ->
assertEquals(expected, actual)
}
}
}
private fun unpackTest(name: String, expected: JagArchive) {
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { buf ->
JagArchive.unpack(buf).use { actual ->
assertEquals(expected, actual)
}
}
}
private companion object {
private val ROOT = Path.of(JagArchiveTest::class.java.getResource("jag").toURI())
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 8 B

Loading…
Cancel
Save