forked from openrs2/openrs2
parent
3734d4709b
commit
26c6ad21e3
@ -0,0 +1,293 @@ |
||||
package org.openrs2.cache |
||||
|
||||
import io.netty.buffer.ByteBuf |
||||
import io.netty.buffer.ByteBufAllocator |
||||
import io.netty.buffer.ByteBufInputStream |
||||
import io.netty.buffer.ByteBufOutputStream |
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap |
||||
import org.openrs2.buffer.use |
||||
import org.openrs2.compress.bzip2.Bzip2 |
||||
import org.openrs2.util.jagHashCode |
||||
import java.io.Closeable |
||||
import java.io.FileNotFoundException |
||||
|
||||
/** |
||||
* An interface for reading and writing `.jag` archives, which are used by |
||||
* RuneScape Classic and early versions of RuneScape 2. |
||||
* |
||||
* Unlike the client, this implementation is case-sensitive. Entry names should |
||||
* therefore be supplied in uppercase for compatibility. |
||||
* |
||||
* This class is not thread safe. |
||||
*/ |
||||
public class JagArchive : Closeable { |
||||
private val entries = Int2ObjectLinkedOpenHashMap<ByteBuf>() |
||||
|
||||
/** |
||||
* The number of entries in the archive. |
||||
*/ |
||||
public val size: Int |
||||
get() = entries.size |
||||
|
||||
/** |
||||
* Lists all entries in the archive. |
||||
* @return a sorted list of entry name hashes. |
||||
*/ |
||||
public fun list(): Iterator<Int> { |
||||
return entries.keys.iterator() |
||||
} |
||||
|
||||
/** |
||||
* Checks whether an entry exists. |
||||
* @param name the entry's name. |
||||
* @return `true` if so, `false` otherwise. |
||||
*/ |
||||
public fun exists(name: String): Boolean { |
||||
return existsNamed(name.jagHashCode()) |
||||
} |
||||
|
||||
/** |
||||
* Checks whether an entry exists. |
||||
* @param nameHash the entry's name hash. |
||||
* @return `true` if so, `false` otherwise. |
||||
*/ |
||||
public fun existsNamed(nameHash: Int): Boolean { |
||||
return entries.containsKey(nameHash) |
||||
} |
||||
|
||||
/** |
||||
* Reads an entry. |
||||
* |
||||
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||
* responsibility to release the [ByteBuf]. |
||||
* @param name the entry's name. |
||||
* @return the contents of the entry. |
||||
* @throws FileNotFoundException if the entry does not exist. |
||||
*/ |
||||
public fun read(name: String): ByteBuf { |
||||
return readNamed(name.jagHashCode()) |
||||
} |
||||
|
||||
/** |
||||
* Reads an entry. |
||||
* |
||||
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||
* responsibility to release the [ByteBuf]. |
||||
* @param nameHash the entry's name hash. |
||||
* @return the contents of the entry. |
||||
* @throws FileNotFoundException if the entry does not exist. |
||||
*/ |
||||
public fun readNamed(nameHash: Int): ByteBuf { |
||||
val buf = entries[nameHash] ?: throw FileNotFoundException() |
||||
return buf.retainedSlice() |
||||
} |
||||
|
||||
/** |
||||
* Writes an entry. If an entry with the same name hash already exists, it |
||||
* is overwritten. |
||||
* @param name the entry's name. |
||||
* @param buf the new contents of the entry. |
||||
*/ |
||||
public fun write(name: String, buf: ByteBuf) { |
||||
writeNamed(name.jagHashCode(), buf) |
||||
} |
||||
|
||||
/** |
||||
* Writes an entry. If an entry with the same name hash already exists, it |
||||
* is overwritten. |
||||
* @param nameHash the entry's name hash. |
||||
* @param buf the new contents of the entry. |
||||
*/ |
||||
public fun writeNamed(nameHash: Int, buf: ByteBuf) { |
||||
entries.put(nameHash, buf.copy().asReadOnly())?.release() |
||||
} |
||||
|
||||
/** |
||||
* Deletes an entry. Does nothing if the entry does not exist. |
||||
* @param name the entry's name. |
||||
*/ |
||||
public fun remove(name: String) { |
||||
removeNamed(name.jagHashCode()) |
||||
} |
||||
|
||||
/** |
||||
* Deletes an entry. Does nothing if the entry does not exist. |
||||
* @param nameHash the entry's name hash. |
||||
*/ |
||||
public fun removeNamed(nameHash: Int) { |
||||
entries.remove(nameHash)?.release() |
||||
} |
||||
|
||||
/** |
||||
* Packs a `.jag` archive into a compressed [ByteBuf] using the given |
||||
* compression method. |
||||
* |
||||
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||
* responsibility to release the [ByteBuf]. |
||||
* @param compressedArchive `true` if the archive should be compressed as a |
||||
* whole, `false` if each entry should be compressed individually. |
||||
* @param alloc the allocator. |
||||
* @return the compressed archive. |
||||
*/ |
||||
public fun pack(compressedArchive: Boolean, alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { |
||||
alloc.buffer().use { output -> |
||||
alloc.buffer().use { uncompressedArchiveBuf -> |
||||
uncompressedArchiveBuf.writeShort(size) |
||||
var index = uncompressedArchiveBuf.writerIndex() + size * 10 |
||||
|
||||
for ((nameHash, uncompressedEntryBuf) in entries) { |
||||
uncompressedArchiveBuf.writeInt(nameHash) |
||||
uncompressedArchiveBuf.writeMedium(uncompressedEntryBuf.readableBytes()) |
||||
|
||||
compress(uncompressedEntryBuf.slice(), !compressedArchive).use { entryBuf -> |
||||
val entryLen = entryBuf.readableBytes() |
||||
uncompressedArchiveBuf.writeMedium(entryLen) |
||||
|
||||
uncompressedArchiveBuf.setBytes(index, entryBuf) |
||||
index += entryLen |
||||
} |
||||
} |
||||
|
||||
uncompressedArchiveBuf.writerIndex(index) |
||||
|
||||
val uncompressedLen = uncompressedArchiveBuf.readableBytes() |
||||
output.writeMedium(uncompressedLen) |
||||
|
||||
compress(uncompressedArchiveBuf, compressedArchive).use { archiveBuf -> |
||||
val len = archiveBuf.readableBytes() |
||||
if (compressedArchive && uncompressedLen == len) { |
||||
/* |
||||
* This is a bit of an odd corner case. If whole |
||||
* archive compression is enabled and the lengths are |
||||
* equal we have to use individual entry compression |
||||
* instead, as the lengths being equal signals to the |
||||
* client that this mode should be used. |
||||
* |
||||
* If anyone finds a suitable test case for this case, |
||||
* I'd love to see it! |
||||
*/ |
||||
return pack(false, alloc) |
||||
} |
||||
|
||||
output.writeMedium(len) |
||||
output.writeBytes(archiveBuf) |
||||
} |
||||
|
||||
return output.retain() |
||||
} |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* Packs a `.jag` archive into a compressed [ByteBuf]. The best compression |
||||
* method for minimising the size of the compressed archive is |
||||
* automatically selected. Note that this does not necessarily correspond |
||||
* to the minimimal amount of RAM usage at runtime. |
||||
* |
||||
* This method allocates and returns a new [ByteBuf]. It is the caller's |
||||
* responsibility to release the [ByteBuf]. |
||||
* @param alloc the allocator. |
||||
* @return the compressed archive. |
||||
*/ |
||||
public fun packBest(alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT): ByteBuf { |
||||
pack(true, alloc).use { compressedArchive -> |
||||
pack(false, alloc).use { compressedEntries -> |
||||
// If equal, pick the archive that only requires one |
||||
// decompression instead of several to save CPU time. |
||||
return if (compressedArchive.readableBytes() <= compressedEntries.readableBytes()) { |
||||
compressedArchive.retain() |
||||
} else { |
||||
compressedEntries.retain() |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
override fun close() { |
||||
entries.values.forEach(ByteBuf::release) |
||||
} |
||||
|
||||
override fun equals(other: Any?): Boolean { |
||||
if (this === other) return true |
||||
if (javaClass != other?.javaClass) return false |
||||
|
||||
other as JagArchive |
||||
|
||||
if (entries != other.entries) return false |
||||
|
||||
return true |
||||
} |
||||
|
||||
override fun hashCode(): Int { |
||||
return entries.hashCode() |
||||
} |
||||
|
||||
public companion object { |
||||
/** |
||||
* Unpacks a [ByteBuf] containing a compressed `.jag` archive. |
||||
* @param buf the compressed archive. |
||||
* @return the unpacked archive. |
||||
*/ |
||||
public fun unpack(buf: ByteBuf): JagArchive { |
||||
val archive = JagArchive() |
||||
|
||||
val uncompressedLen = buf.readUnsignedMedium() |
||||
val len = buf.readUnsignedMedium() |
||||
|
||||
val compressedArchive = len != uncompressedLen |
||||
|
||||
uncompress(buf, compressedArchive, len, uncompressedLen).use { archiveBuf -> |
||||
val size = archiveBuf.readUnsignedShort() |
||||
var index = archiveBuf.readerIndex() + size * 10 |
||||
|
||||
for (id in 0 until size) { |
||||
val nameHash = archiveBuf.readInt() |
||||
val uncompressedEntryLen = archiveBuf.readUnsignedMedium() |
||||
val entryLen = archiveBuf.readUnsignedMedium() |
||||
|
||||
val entry = archiveBuf.slice(index, entryLen) |
||||
|
||||
uncompress(entry, !compressedArchive, entryLen, uncompressedEntryLen).use { entryBuf -> |
||||
if (!archive.existsNamed(nameHash)) { |
||||
// Store the first entry if there is a collision, |
||||
// for compatibility with the client. |
||||
archive.writeNamed(nameHash, entryBuf) |
||||
} |
||||
} |
||||
|
||||
index += entryLen |
||||
} |
||||
} |
||||
|
||||
return archive |
||||
} |
||||
|
||||
private fun compress(input: ByteBuf, compressed: Boolean): ByteBuf { |
||||
return if (compressed) { |
||||
input.alloc().buffer().use { output -> |
||||
Bzip2.createHeaderlessOutputStream(ByteBufOutputStream(output)).use { stream -> |
||||
input.readBytes(stream, input.readableBytes()) |
||||
} |
||||
|
||||
output.retain() |
||||
} |
||||
} else { |
||||
input.readRetainedSlice(input.readableBytes()) |
||||
} |
||||
} |
||||
|
||||
private fun uncompress(buf: ByteBuf, compressed: Boolean, compressedLen: Int, uncompressedLen: Int): ByteBuf { |
||||
return if (compressed) { |
||||
buf.alloc().buffer(uncompressedLen, uncompressedLen).use { output -> |
||||
Bzip2.createHeaderlessInputStream(ByteBufInputStream(buf.readSlice(compressedLen))).use { stream -> |
||||
output.writeBytes(stream, uncompressedLen) |
||||
} |
||||
|
||||
output.retain() |
||||
} |
||||
} else { |
||||
buf.readRetainedSlice(compressedLen) |
||||
} |
||||
} |
||||
} |
||||
} |
@ -0,0 +1,198 @@ |
||||
package org.openrs2.cache |
||||
|
||||
import io.netty.buffer.Unpooled |
||||
import org.openrs2.buffer.copiedBuffer |
||||
import org.openrs2.buffer.use |
||||
import org.openrs2.util.jagHashCode |
||||
import java.nio.file.Files |
||||
import java.nio.file.Path |
||||
import kotlin.test.Test |
||||
import kotlin.test.assertEquals |
||||
import kotlin.test.assertFalse |
||||
import kotlin.test.assertTrue |
||||
|
||||
class JagArchiveTest { |
||||
@Test |
||||
fun testEmpty() { |
||||
JagArchive().use { archive -> |
||||
assertEquals(0, archive.size) |
||||
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||
|
||||
packTest("empty-compressed-archive.jag", archive, true) |
||||
packTest("empty-compressed-entries.jag", archive, false) |
||||
packBestTest("empty-compressed-entries.jag", archive) |
||||
|
||||
unpackTest("empty-compressed-archive.jag", archive) |
||||
unpackTest("empty-compressed-entries.jag", archive) |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
fun testSingleEntry() { |
||||
JagArchive().use { archive -> |
||||
assertEquals(0, archive.size) |
||||
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||
assertFalse(archive.exists("TEST.TXT")) |
||||
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
|
||||
copiedBuffer("OpenRS2").use { expected -> |
||||
archive.write("TEST.TXT", expected) |
||||
|
||||
archive.read("TEST.TXT").use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
|
||||
archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
|
||||
assertEquals(1, archive.size) |
||||
assertEquals(listOf("TEST.TXT".jagHashCode()), archive.list().asSequence().toList()) |
||||
assertTrue(archive.exists("TEST.TXT")) |
||||
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("HELLO.TXT")) |
||||
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
|
||||
packTest("single-compressed-archive.jag", archive, true) |
||||
packTest("single-compressed-entries.jag", archive, false) |
||||
packBestTest("single-compressed-entries.jag", archive) |
||||
|
||||
unpackTest("single-compressed-archive.jag", archive) |
||||
unpackTest("single-compressed-entries.jag", archive) |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
fun testMultipleEntries() { |
||||
JagArchive().use { archive -> |
||||
assertEquals(0, archive.size) |
||||
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||
assertFalse(archive.exists("TEST.TXT")) |
||||
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("HELLO.TXT")) |
||||
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
|
||||
copiedBuffer("OpenRS2").use { expected -> |
||||
archive.write("TEST.TXT", expected) |
||||
|
||||
archive.read("TEST.TXT").use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
|
||||
archive.readNamed("TEST.TXT".jagHashCode()).use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
|
||||
copiedBuffer("Hello").use { expected -> |
||||
archive.write("HELLO.TXT", expected) |
||||
|
||||
archive.read("HELLO.TXT").use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
|
||||
archive.readNamed("HELLO.TXT".jagHashCode()).use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
|
||||
assertEquals(2, archive.size) |
||||
assertEquals( |
||||
listOf( |
||||
"TEST.TXT".jagHashCode(), |
||||
"HELLO.TXT".jagHashCode() |
||||
), archive.list().asSequence().toList() |
||||
) |
||||
assertTrue(archive.exists("TEST.TXT")) |
||||
assertTrue(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertTrue(archive.exists("HELLO.TXT")) |
||||
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("OTHER.TXT")) |
||||
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||
|
||||
packTest("multiple-compressed-archive.jag", archive, true) |
||||
packTest("multiple-compressed-entries.jag", archive, false) |
||||
packBestTest("multiple-compressed-archive.jag", archive) |
||||
|
||||
unpackTest("multiple-compressed-archive.jag", archive) |
||||
unpackTest("multiple-compressed-entries.jag", archive) |
||||
|
||||
archive.remove("TEST.TXT") |
||||
|
||||
assertEquals(1, archive.size) |
||||
assertEquals(listOf("HELLO.TXT".jagHashCode()), archive.list().asSequence().toList()) |
||||
assertFalse(archive.exists("TEST.TXT")) |
||||
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertTrue(archive.exists("HELLO.TXT")) |
||||
assertTrue(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("OTHER.TXT")) |
||||
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||
|
||||
archive.remove("TEST.TXT") |
||||
archive.removeNamed("HELLO.TXT".jagHashCode()) // check remove a non-existent entry works |
||||
|
||||
assertEquals(0, archive.size) |
||||
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||
assertFalse(archive.exists("TEST.TXT")) |
||||
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("HELLO.TXT")) |
||||
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("OTHER.TXT")) |
||||
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||
|
||||
archive.removeNamed("HELLO.TXT".jagHashCode()) |
||||
|
||||
archive.remove("OTHER.TXT") // check removing an entry that never existed works |
||||
archive.removeNamed("OTHER.TXT".jagHashCode()) |
||||
|
||||
assertEquals(0, archive.size) |
||||
assertEquals(emptyList(), archive.list().asSequence().toList()) |
||||
assertFalse(archive.exists("TEST.TXT")) |
||||
assertFalse(archive.existsNamed("TEST.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("HELLO.TXT")) |
||||
assertFalse(archive.existsNamed("HELLO.TXT".jagHashCode())) |
||||
assertFalse(archive.exists("OTHER.TXT")) |
||||
assertFalse(archive.existsNamed("OTHER.TXT".jagHashCode())) |
||||
} |
||||
} |
||||
|
||||
@Test |
||||
fun testDuplicateEntries() { |
||||
JagArchive().use { archive -> |
||||
copiedBuffer("OpenRS2").use { buf -> |
||||
archive.write("TEST.TXT", buf) |
||||
} |
||||
|
||||
unpackTest("duplicate-entries.jag", archive) |
||||
} |
||||
} |
||||
|
||||
private fun packTest(name: String, archive: JagArchive, compressedArchive: Boolean) { |
||||
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> |
||||
archive.pack(compressedArchive).use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
} |
||||
|
||||
private fun packBestTest(name: String, archive: JagArchive) { |
||||
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { expected -> |
||||
archive.packBest().use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
} |
||||
|
||||
private fun unpackTest(name: String, expected: JagArchive) { |
||||
Unpooled.wrappedBuffer(Files.readAllBytes(ROOT.resolve(name))).use { buf -> |
||||
JagArchive.unpack(buf).use { actual -> |
||||
assertEquals(expected, actual) |
||||
} |
||||
} |
||||
} |
||||
|
||||
private companion object { |
||||
private val ROOT = Path.of(JagArchiveTest::class.java.getResource("jag").toURI()) |
||||
} |
||||
} |
Binary file not shown.
After Width: | Height: | Size: 39 B |
After Width: | Height: | Size: 8 B |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
Reference in new issue