forked from openrs2/openrs2
parent
3734d4709b
commit
26c6ad21e3
@ -0,0 +1,293 @@ |
|||||||
|
package org.openrs2.cache |
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf |
||||||
|
import io.netty.buffer.ByteBufAllocator |
||||||
|
import io.netty.buffer.ByteBufInputStream |
||||||
|
import io.netty.buffer.ByteBufOutputStream |
||||||
|
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap |
||||||
|
import org.openrs2.buffer.use |
||||||
|
import org.openrs2.compress.bzip2.Bzip2 |
||||||
|
import org.openrs2.util.jagHashCode |
||||||
|
import java.io.Closeable |
||||||
|
import java.io.FileNotFoundException |
||||||
|
|
||||||
|
/** |
||||||
|
* An interface for reading and writing `.jag` archives, which are used by |
||||||
|
* RuneScape Classic and early versions of RuneScape 2. |
||||||
|
* |
||||||
|
* Unlike the client, this implementation is case-sensitive. Entry names should |
||||||
|
* therefore be supplied in uppercase for compatibility. |
||||||
|
* |
||||||
|
* This class is not thread safe. |
||||||
|
*/ |
||||||
|
public class JagArchive : Closeable { |
||||||
|
private val entries = Int2ObjectLinkedOpenHashMap<ByteBuf>() |
||||||
|
|
||||||
|
/** |
||||||
|
* The number of entries in the archive. |
||||||
|
*/ |
||||||
|
public val size: Int |
||||||
|
get() = entries.size |
||||||
|
|
||||||
|
/** |
||||||
|
* Lists all entries in the archive. |
||||||
|
* @return a sorted list of entry name hashes. |
||||||
|
*/ |
||||||
|
public fun list(): Iterator<Int> { |
||||||
|
return entries.keys.iterator() |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks whether an entry exists. |
||||||
|
* @param name the entry's name. |
||||||
|
* @return `true` if so, `false` otherwise. |
||||||
|
*/ |
||||||
|
public fun exists(name: String): Boolean { |
||||||
|
return existsNamed(name.jagHashCode()) |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Checks whether an entry exists. |
||||||
|
* @param nameHash the entry's name hash. |
||||||
|
* @return `true` if so, `false` otherwise. |
||||||
|
*/ |
||||||
|
public fun existsNamed(nameHash: Int): Boolean { |
||||||
|
return entries.containsKey(nameHash) |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Reads an entry. |
||||||
|
* |
||||||
|
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||||
|
* responsibility to release the [ByteBuf]. |
||||||
|
* @param name the entry's name. |
||||||
|
* @return the contents of the entry. |
||||||
|
* @throws FileNotFoundException if the entry does not exist. |
||||||
|
*/ |
||||||
|
public fun read(name: String): ByteBuf { |
||||||
|
return readNamed(name.jagHashCode()) |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Reads an entry. |
||||||
|
* |
||||||
|
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||||
|
* responsibility to release the [ByteBuf]. |
||||||
|
* @param nameHash the entry's name hash. |
||||||
|
* @return the contents of the entry. |
||||||
|
* @throws FileNotFoundException if the entry does not exist. |
||||||
|
*/ |
||||||
|
public fun readNamed(nameHash: Int): ByteBuf { |
||||||
|
val buf = entries[nameHash] ?: throw FileNotFoundException() |
||||||
|
return buf.retainedSlice() |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes an entry. If an entry with the same name hash already exists, it |
||||||
|
* is overwritten. |
||||||
|
* @param name the entry's name. |
||||||
|
* @param buf the new contents of the entry. |
||||||
|
*/ |
||||||
|
public fun write(name: String, buf: ByteBuf) { |
||||||
|
writeNamed(name.jagHashCode(), buf) |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Writes an entry. If an entry with the same name hash already exists, it |
||||||
|
* is overwritten. |
||||||
|
* @param nameHash the entry's name hash. |
||||||
|
* @param buf the new contents of the entry. |
||||||
|
*/ |
||||||
|
public fun writeNamed(nameHash: Int, buf: ByteBuf) { |
||||||
|
entries.put(nameHash, buf.copy().asReadOnly())?.release() |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Deletes an entry. Does nothing if the entry does not exist. |
||||||
|
* @param name the entry's name. |
||||||
|
*/ |
||||||
|
public fun remove(name: String) { |
||||||
|
removeNamed(name.jagHashCode()) |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Deletes an entry. Does nothing if the entry does not exist. |
||||||
|
* @param nameHash the entry's name hash. |
||||||
|
*/ |
||||||
|
public fun removeNamed(nameHash: Int) { |
||||||
|
entries.remove(nameHash)?.release() |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Packs a `.jag` archive into a compressed [ByteBuf] using the given |
||||||
|
* compression method. |
||||||
|
* |
||||||
|
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||||
|
* responsibility to release the [ByteBuf]. |
||||||
|
* @param compressedArchive `true` if the archive should be compressed as a |
||||||
|
* whole, `false` if each entry should be compressed individually. |
||||||
|
* @param alloc the allocator. |
||||||
|
* @return the compressed archive. |
||||||
|
*/ |
||||||
|
public fun pack(compressedArchive: Boolean, alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { |
||||||
|
alloc.buffer().use { output -> |
||||||
|
alloc.buffer().use { uncompressedArchiveBuf -> |
||||||
|
uncompressedArchiveBuf.writeShort(size) |
||||||
|
var index = uncompressedArchiveBuf.writerIndex() + size * 10 |
||||||
|
|
||||||
|
for ((nameHash, uncompressedEntryBuf) in entries) { |
||||||
|
uncompressedArchiveBuf.writeInt(nameHash) |
||||||
|
uncompressedArchiveBuf.writeMedium(uncompressedEntryBuf.readableBytes()) |
||||||
|
|
||||||
|
compress(uncompressedEntryBuf.slice(), !compressedArchive).use { entryBuf -> |
||||||
|
val entryLen = entryBuf.readableBytes() |
||||||
|
uncompressedArchiveBuf.writeMedium(entryLen) |
||||||
|
|
||||||
|
uncompressedArchiveBuf.setBytes(index, entryBuf) |
||||||
|
index += entryLen |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
uncompressedArchiveBuf.writerIndex(index) |
||||||
|
|
||||||
|
val uncompressedLen = uncompressedArchiveBuf.readableBytes() |
||||||
|
output.writeMedium(uncompressedLen) |
||||||
|
|
||||||
|
compress(uncompressedArchiveBuf, compressedArchive).use { archiveBuf -> |
||||||
|
val len = archiveBuf.readableBytes() |
||||||
|
if (compressedArchive && uncompressedLen == len) { |
||||||
|
/* |
||||||
|
* This is a bit of an odd corner case. If whole |
||||||
|
* archive compression is enabled and the lengths are |
||||||
|
* equal we have to use individual entry compression |
||||||
|
* instead, as the lengths being equal signals to the |
||||||
|
* client that this mode should be used. |
||||||
|
* |
||||||
|
* If anyone finds a suitable test case for this case, |
||||||
|
* I'd love to see it! |
||||||
|
*/ |
||||||
|
return pack(false, alloc) |
||||||
|
} |
||||||
|
|
||||||
|
output.writeMedium(len) |
||||||
|
output.writeBytes(archiveBuf) |
||||||
|
} |
||||||
|
|
||||||
|
return output.retain() |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Packs a `.jag` archive into a compressed [ByteBuf]. The best compression |
||||||
|
* method for minimising the size of the compressed archive is |
||||||
|
* automatically selected. Note that this does not necessarily correspond |
||||||
|
* to the minimimal amount of RAM usage at runtime. |
||||||
|
* |
||||||
|
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||||
|
* responsibility to release the [ByteBuf]. |
||||||
|
* @param alloc the allocator. |
||||||
|
* @return the compressed archive. |
||||||
|
*/ |
||||||
|
public fun packBest(alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { |
||||||
|
pack(true, alloc).use { compressedArchive -> |
||||||
|
pack(false, alloc).use { compressedEntries -> |
||||||
|
// If equal, pick the archive that only requires one |
||||||
|
// decompression instead of several to save CPU time. |
||||||
|
return if (compressedArchive.readableBytes() <= compressedEntries.readableBytes()) { |
||||||
|
compressedArchive.retain() |
||||||
|
} else { |
||||||
|
compressedEntries.retain() |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
override fun close() { |
||||||
|
entries.values.forEach(ByteBuf::release) |
||||||
|
} |
||||||
|
|
||||||
|
override fun equals(other: Any?): Boolean { |
||||||
|
if (this === other) return true |
||||||
|
if (javaClass != other?.javaClass) return false |
||||||
|
|
||||||
|
other as JagArchive |
||||||
|
|
||||||
|
if (entries != other.entries) return false |
||||||
|
|
||||||
|
return true |
||||||
|
} |
||||||
|
|
||||||
|
override fun hashCode(): Int { |
||||||
|
return entries.hashCode() |
||||||
|
} |
||||||
|
|
||||||
|
public companion object { |
||||||
|
/** |
||||||
|
* Unpacks a [ByteBuf] containing a compressed `.jag` archive. |
||||||
|
* @param buf the compressed archive. |
||||||
|
* @return the unpacked archive. |
||||||
|
*/ |
||||||
|
public fun unpack(buf: ByteBuf): JagArchive { |
||||||
|
val archive = JagArchive() |
||||||
|
|
||||||
|
val uncompressedLen = buf.readUnsignedMedium() |
||||||
|
val len = buf.readUnsignedMedium() |
||||||
|
|
||||||
|
val compressedArchive = len != uncompressedLen |
||||||
|
|
||||||
|
uncompress(buf, compressedArchive, len, uncompressedLen).use { archiveBuf -> |
||||||
|
val size = archiveBuf.readUnsignedShort() |
||||||
|
var index = archiveBuf.readerIndex() + size * 10 |
||||||
|
|
||||||
|
for (id in 0 until size) { |
||||||
|
val nameHash = archiveBuf.readInt() |
||||||
|
val uncompressedEntryLen = archiveBuf.readUnsignedMedium() |
||||||
|
val entryLen = archiveBuf.readUnsignedMedium() |
||||||
|
|
||||||
|
val entry = archiveBuf.slice(index, entryLen) |
||||||
|
|
||||||
|
uncompress(entry, !compressedArchive, entryLen, uncompressedEntryLen).use { entryBuf -> |
||||||
|
if (!archive.existsNamed(nameHash)) { |
||||||
|
// Store the first entry if there is a collision, |
||||||
|
// for compatibility with the client. |
||||||
|
archive.writeNamed(nameHash, entryBuf) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
index += entryLen |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return archive |
||||||
|
} |
||||||
|
|
||||||
|
private fun compress(input: ByteBuf, compressed: Boolean): ByteBuf { |
||||||
|
return if (compressed) { |
||||||
|
input.alloc().buffer().use { output -> |
||||||
|
Bzip2.createHeaderlessOutputStream(ByteBufOutputStream(output)).use { stream -> |
||||||
|
input.readBytes(stream, input.readableBytes()) |
||||||
|
} |
||||||
|
|
||||||
|
output.retain() |
||||||
|
} |
||||||
|
} else { |
||||||
|
input.readRetainedSlice(input.readableBytes()) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun uncompress(buf: ByteBuf, compressed: Boolean, compressedLen: Int, uncompressedLen: Int): ByteBuf { |
||||||
|
return if (compressed) { |
||||||
|
buf.alloc().buffer(uncompressedLen, uncompressedLen).use { output -> |
||||||
|
Bzip2.createHeaderlessInputStream(ByteBufInputStream(buf.readSlice(compressedLen))).use { stream -> |
||||||
|
output.writeBytes(stream, uncompressedLen) |
||||||
|
} |
||||||
|
|
||||||
|
output.retain() |
||||||
|
} |
||||||
|
} else { |
||||||
|
buf.readRetainedSlice(compressedLen) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,198 @@ |
|||||||
|
package org.openrs2.cache |
||||||
|
|
||||||
|
import io.netty.buffer.Unpooled |
||||||
|
import org.openrs2.buffer.copiedBuffer |
||||||
|
import org.openrs2.buffer.use |
||||||
|
import org.openrs2.util.jagHashCode |
||||||
|
import java.nio.file.Files |
||||||
|
import java.nio.file.Path |
||||||
|
import kotlin.test.Test |
||||||
|
import kotlin.test.assertEquals |
||||||
|
import kotlin.test.assertFalse |
||||||
|
import kotlin.test.assertTrue |
||||||
|
|
||||||
|
class JagArchiveTest { |
||||||
|
@Test |
||||||
|
fun testEmpty() { |
||||||
|
JagArchive().use { archive -> |
||||||
|
assertEquals(0, archive.size) |
||||||
|
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||||
|
|
||||||
|
packTest("empty-compressed-archive.jag", archive, true) |
||||||
|
packTest("empty-compressed-entries.jag", archive, false) |
||||||
|
packBestTest("empty-compressed-entries.jag", archive) |
||||||
|
|
||||||
|
unpackTest("empty-compressed-archive.jag", archive) |
||||||
|
unpackTest("empty-compressed-entries.jag", archive) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
fun testSingleEntry() { |
||||||
|
JagArchive().use { archive -> |
||||||
|
assertEquals(0, archive.size) |
||||||
|
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||||
|
assertFalse(archive.exists("TEST.TXT")) |
||||||
|
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
|
||||||
|
copiedBuffer("OpenRS2").use { expected -> |
||||||
|
archive.write("TEST.TXT", expected) |
||||||
|
|
||||||
|
archive.read("TEST.TXT").use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
|
||||||
|
archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
assertEquals(1, archive.size) |
||||||
|
assertEquals(listOf("TEST.TXT".jagHashCode()), archive.list().asSequence().toList()) |
||||||
|
assertTrue(archive.exists("TEST.TXT")) |
||||||
|
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("HELLO.TXT")) |
||||||
|
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
|
||||||
|
packTest("single-compressed-archive.jag", archive, true) |
||||||
|
packTest("single-compressed-entries.jag", archive, false) |
||||||
|
packBestTest("single-compressed-entries.jag", archive) |
||||||
|
|
||||||
|
unpackTest("single-compressed-archive.jag", archive) |
||||||
|
unpackTest("single-compressed-entries.jag", archive) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
fun testMultipleEntries() { |
||||||
|
JagArchive().use { archive -> |
||||||
|
assertEquals(0, archive.size) |
||||||
|
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||||
|
assertFalse(archive.exists("TEST.TXT")) |
||||||
|
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("HELLO.TXT")) |
||||||
|
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
|
||||||
|
copiedBuffer("OpenRS2").use { expected -> |
||||||
|
archive.write("TEST.TXT", expected) |
||||||
|
|
||||||
|
archive.read("TEST.TXT").use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
|
||||||
|
archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
copiedBuffer("Hello").use { expected -> |
||||||
|
archive.write("HELLO.TXT", expected) |
||||||
|
|
||||||
|
archive.read("HELLO.TXT").use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
|
||||||
|
archive.readNamed("HELLO.TXT".jagHashCode()).use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
assertEquals(2, archive.size) |
||||||
|
assertEquals( |
||||||
|
listOf( |
||||||
|
"TEST.TXT".jagHashCode(), |
||||||
|
"HELLO.TXT".jagHashCode() |
||||||
|
), archive.list().asSequence().toList() |
||||||
|
) |
||||||
|
assertTrue(archive.exists("TEST.TXT")) |
||||||
|
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertTrue(archive.exists("HELLO.TXT")) |
||||||
|
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("OTHER.TXT")) |
||||||
|
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||||
|
|
||||||
|
packTest("multiple-compressed-archive.jag", archive, true) |
||||||
|
packTest("multiple-compressed-entries.jag", archive, false) |
||||||
|
packBestTest("multiple-compressed-archive.jag", archive) |
||||||
|
|
||||||
|
unpackTest("multiple-compressed-archive.jag", archive) |
||||||
|
unpackTest("multiple-compressed-entries.jag", archive) |
||||||
|
|
||||||
|
archive.remove("TEST.TXT") |
||||||
|
|
||||||
|
assertEquals(1, archive.size) |
||||||
|
assertEquals(listOf("HELLO.TXT".jagHashCode()), archive.list().asSequence().toList()) |
||||||
|
assertFalse(archive.exists("TEST.TXT")) |
||||||
|
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertTrue(archive.exists("HELLO.TXT")) |
||||||
|
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("OTHER.TXT")) |
||||||
|
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||||
|
|
||||||
|
archive.remove("TEST.TXT") |
||||||
|
archive.removeNamed("HELLO.TXT".jagHashCode()) // check remove a non-existent entry works |
||||||
|
|
||||||
|
assertEquals(0, archive.size) |
||||||
|
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||||
|
assertFalse(archive.exists("TEST.TXT")) |
||||||
|
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("HELLO.TXT")) |
||||||
|
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("OTHER.TXT")) |
||||||
|
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||||
|
|
||||||
|
archive.removeNamed("HELLO.TXT".jagHashCode()) |
||||||
|
|
||||||
|
archive.remove("OTHER.TXT") // check removing an entry that never existed works |
||||||
|
archive.removeNamed("OTHER.TXT".jagHashCode()) |
||||||
|
|
||||||
|
assertEquals(0, archive.size) |
||||||
|
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||||
|
assertFalse(archive.exists("TEST.TXT")) |
||||||
|
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("HELLO.TXT")) |
||||||
|
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||||
|
assertFalse(archive.exists("OTHER.TXT")) |
||||||
|
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
@Test |
||||||
|
fun testDuplicateEntries() { |
||||||
|
JagArchive().use { archive -> |
||||||
|
copiedBuffer("OpenRS2").use { buf -> |
||||||
|
archive.write("TEST.TXT", buf) |
||||||
|
} |
||||||
|
|
||||||
|
unpackTest("duplicate-entries.jag", archive) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun packTest(name: String, archive: JagArchive, compressedArchive: Boolean) { |
||||||
|
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> |
||||||
|
archive.pack(compressedArchive).use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun packBestTest(name: String, archive: JagArchive) { |
||||||
|
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> |
||||||
|
archive.packBest().use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun unpackTest(name: String, expected: JagArchive) { |
||||||
|
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { buf -> |
||||||
|
JagArchive.unpack(buf).use { actual -> |
||||||
|
assertEquals(expected, actual) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private companion object { |
||||||
|
private val ROOT = Path.of(JagArchiveTest::class.java.getResource("jag").toURI()) |
||||||
|
} |
||||||
|
} |
Binary file not shown.
After Width: | Height: | Size: 39 B |
After Width: | Height: | Size: 8 B |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in new issue