Add initial high-level cache API

It supports reading and writing a cache backed by a Store, as well as
standalone .js5 files.

I'm not quite as happy with this as I am with the lower-level API yet,
and there are still a few remaining TODOs.

Signed-off-by: Graham <gpe@openrs2.org>

cache/src/main/kotlin/org/openrs2/cache/Archive.kt

@@ -0,0 +1,347 @@
+package org.openrs2.cache
+
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufAllocator
+import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
+import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
+import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMaps
+import org.openrs2.buffer.crc32
+import org.openrs2.buffer.use
+import org.openrs2.crypto.XteaKey
+import org.openrs2.crypto.whirlpool
+import org.openrs2.util.krHashCode
+import java.io.FileNotFoundException
+import java.io.Flushable
+
+public abstract class Archive internal constructor(
+    protected val alloc: ByteBufAllocator,
+    protected val index: Js5Index,
+    protected val archive: Int,
+    internal val unpackedCache: UnpackedCache
+) : Flushable {
+    private var dirty = false
+
+    internal inner class Unpacked(
+        private val entry: Js5Index.MutableGroup,
+        val key: XteaKey,
+        private var files: Int2ObjectSortedMap<ByteBuf>
+    ) {
+        private var dirty = false
+
+        private fun ensureWritable() {
+            if (files.size == 1 && files is Int2ObjectSortedMaps.Singleton) {
+                files = Int2ObjectAVLTreeMap(files)
+            }
+        }
+
+        fun read(file: Int): ByteBuf {
+            val fileEntry = entry[file] ?: throw FileNotFoundException()
+            return files[fileEntry.id]!!.retainedSlice()
+        }
+
+        fun readNamed(fileNameHash: Int): ByteBuf {
+            val fileEntry = entry.getNamed(fileNameHash) ?: throw FileNotFoundException()
+            return files[fileEntry.id]!!.retainedSlice()
+        }
+
+        fun write(file: Int, buf: ByteBuf) {
+            ensureWritable()
+
+            val fileEntry = entry.createOrGet(file)
+            files.put(fileEntry.id, buf.copy().asReadOnly())?.release()
+            dirty = true
+        }
+
+        fun writeNamed(fileNameHash: Int, buf: ByteBuf) {
+            ensureWritable()
+
+            val fileEntry = entry.createOrGetNamed(fileNameHash)
+            files.put(fileEntry.id, buf.copy().asReadOnly())?.release()
+            dirty = true
+        }
+
+        fun remove(file: Int) {
+            ensureWritable()
+
+            val fileEntry = entry.remove(file) ?: return
+            files.remove(fileEntry.id)?.release()
+            dirty = true
+        }
+
+        fun removeNamed(fileNameHash: Int) {
+            ensureWritable()
+
+            val fileEntry = entry.removeNamed(fileNameHash) ?: return
+            files.remove(fileEntry.id)?.release()
+            dirty = true
+        }
+
+        fun flush() {
+            if (!dirty) {
+                return
+            }
+
+            Group.pack(files).use { buf ->
+                if (index.hasLengths) {
+                    entry.uncompressedLength = buf.readableBytes()
+                }
+
+                if (index.hasUncompressedChecksums) {
+                    entry.uncompressedChecksum = buf.crc32()
+                }
+
+                Js5Compression.compressBest(buf, key = key).use { compressed ->
+                    entry.checksum = compressed.crc32()
+
+                    if (index.hasLengths) {
+                        entry.length = compressed.readableBytes()
+                    }
+
+                    if (index.hasDigests) {
+                        entry.digest = compressed.whirlpool()
+                    }
+
+                    appendVersion(buf, ++entry.version)
+                    writePacked(entry.id, compressed)
+                }
+            }
+
+            dirty = false
+        }
+
+        fun release() {
+            files.values.forEach(ByteBuf::release)
+        }
+    }
+
+    // TODO(gpe): rename/move, reindex, rekey, method to go from name->id
+
+    public fun exists(group: Int): Boolean {
+        require(group >= 0)
+        return index.contains(group)
+    }
+
+    public fun existsNamed(groupNameHash: Int): Boolean {
+        return index.containsNamed(groupNameHash)
+    }
+
+    public fun exists(group: String): Boolean {
+        return existsNamed(group.krHashCode())
+    }
+
+    public fun exists(group: Int, file: Int): Boolean {
+        require(group >= 0 && file >= 0)
+
+        val entry = index[group] ?: return false
+        return entry.contains(file)
+    }
+
+    public fun existsNamed(groupNameHash: Int, fileNameHash: Int): Boolean {
+        val entry = index.getNamed(groupNameHash) ?: return false
+        return entry.containsNamed(fileNameHash)
+    }
+
+    public fun exists(group: String, file: String): Boolean {
+        return existsNamed(group.krHashCode(), file.krHashCode())
+    }
+
+    public fun list(): Iterator<Js5Index.Group<*>> {
+        return index.iterator()
+    }
+
+    public fun list(group: Int): Iterator<Js5Index.File> {
+        require(group >= 0)
+
+        val entry = index[group] ?: throw FileNotFoundException()
+        return entry.iterator()
+    }
+
+    public fun listNamed(groupNameHash: Int): Iterator<Js5Index.File> {
+        val entry = index.getNamed(groupNameHash) ?: throw FileNotFoundException()
+        return entry.iterator()
+    }
+
+    public fun list(group: String): Iterator<Js5Index.File> {
+        return listNamed(group.krHashCode())
+    }
+
+    public fun read(group: Int, file: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        require(group >= 0 && file >= 0)
+
+        val entry = index[group] ?: throw FileNotFoundException()
+        val unpacked = getUnpacked(entry, key)
+        return unpacked.read(file)
+    }
+
+    public fun readNamed(groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        val entry = index.getNamed(groupNameHash) ?: throw FileNotFoundException()
+        val unpacked = getUnpacked(entry, key)
+        return unpacked.readNamed(fileNameHash)
+    }
+
+    public fun read(group: String, file: String, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        return readNamed(group.krHashCode(), file.krHashCode(), key)
+    }
+
+    public fun write(group: Int, file: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
+        require(group >= 0 && file >= 0)
+
+        val entry = index.createOrGet(group)
+        val unpacked = createOrGetUnpacked(entry, key, isOverwriting(entry, file))
+        unpacked.write(file, buf)
+
+        dirty = true
+    }
+
+    public fun writeNamed(groupNameHash: Int, fileNameHash: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
+        val entry = index.createOrGetNamed(groupNameHash)
+        val unpacked = createOrGetUnpacked(entry, key, isOverwritingNamed(entry, fileNameHash))
+        unpacked.writeNamed(fileNameHash, buf)
+
+        dirty = true
+        index.hasNames = true
+    }
+
+    public fun write(group: String, file: String, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
+        return writeNamed(group.krHashCode(), file.krHashCode(), buf, key)
+    }
+
+    public fun remove(group: Int) {
+        require(group >= 0)
+
+        val entry = index.remove(group) ?: return
+        unpackedCache.remove(archive, entry.id)
+        removePacked(entry.id)
+
+        dirty = true
+    }
+
+    public fun removeNamed(groupNameHash: Int) {
+        val entry = index.removeNamed(groupNameHash) ?: return
+        unpackedCache.remove(archive, entry.id)
+        removePacked(entry.id)
+
+        dirty = true
+    }
+
+    public fun remove(group: String) {
+        return removeNamed(group.krHashCode())
+    }
+
+    public fun remove(group: Int, file: Int, key: XteaKey = XteaKey.ZERO) {
+        require(group >= 0 && file >= 0)
+
+        val entry = index[group] ?: return
+
+        if (isOverwriting(entry, file)) {
+            remove(group)
+            return
+        }
+
+        val unpacked = getUnpacked(entry, key)
+        unpacked.remove(file)
+
+        dirty = true
+    }
+
+    public fun removeNamed(groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO) {
+        val entry = index.getNamed(groupNameHash) ?: return
+
+        if (isOverwritingNamed(entry, fileNameHash)) {
+            removeNamed(groupNameHash)
+            return
+        }
+
+        val unpacked = getUnpacked(entry, key)
+        unpacked.removeNamed(fileNameHash)
+
+        dirty = true
+    }
+
+    public fun remove(group: String, file: String, key: XteaKey = XteaKey.ZERO) {
+        return removeNamed(group.krHashCode(), file.krHashCode(), key)
+    }
+
+    public override fun flush() {
+        if (!dirty) {
+            return
+        }
+
+        index.version++
+
+        alloc.buffer().use { buf ->
+            index.write(buf)
+
+            Js5Compression.compressBest(buf).use { compressed ->
+                writePackedIndex(compressed)
+            }
+        }
+
+        dirty = false
+    }
+
+    protected abstract fun packedExists(group: Int): Boolean
+    protected abstract fun readPacked(group: Int): ByteBuf
+    protected abstract fun writePacked(group: Int, buf: ByteBuf)
+    protected abstract fun writePackedIndex(buf: ByteBuf)
+    protected abstract fun removePacked(group: Int)
+    protected abstract fun appendVersion(buf: ByteBuf, version: Int)
+    protected abstract fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup)
+    protected abstract fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup)
+
+    private fun isOverwriting(entry: Js5Index.MutableGroup, file: Int): Boolean {
+        val fileEntry = entry.singleOrNull() ?: return false
+        return fileEntry.id == file
+    }
+
+    private fun isOverwritingNamed(entry: Js5Index.MutableGroup, fileNameHash: Int): Boolean {
+        val fileEntry = entry.singleOrNull() ?: return false
+        return fileEntry.nameHash == fileNameHash
+    }
+
+    private fun createOrGetUnpacked(entry: Js5Index.MutableGroup, key: XteaKey, overwrite: Boolean): Unpacked {
+        return if (entry.size == 0 || overwrite) {
+            val unpacked = Unpacked(entry, key, Int2ObjectAVLTreeMap())
+            unpackedCache.put(archive, entry.id, unpacked)
+            return unpacked
+        } else {
+            getUnpacked(entry, key)
+        }
+    }
+
+    private fun getUnpacked(entry: Js5Index.MutableGroup, key: XteaKey): Unpacked {
+        var unpacked = unpackedCache.get(archive, entry.id)
+        if (unpacked != null) {
+            /*
+             * If we've already unpacked the group, we check the programmer
+             * is using the correct key to ensure the code always works,
+             * regardless of group cache size/invalidation behaviour.
+             */
+            require(unpacked.key == key) {
+                "Invalid key for archive $archive group ${entry.id} (expected ${unpacked!!.key}, actual $key)"
+            }
+            return unpacked
+        }
+
+        if (!packedExists(entry.id)) {
+            throw StoreCorruptException("Archive $archive group ${entry.id} is missing")
+        }
+
+        val files = readPacked(entry.id).use { compressed ->
+            // TODO(gpe): check for trailing data?
+            verifyCompressed(compressed, entry)
+
+            Js5Compression.uncompress(compressed, key).use { buf ->
+                verifyUncompressed(buf, entry)
+
+                Group.unpack(buf, entry)
+            }
+        }
+
+        files.replaceAll { _, buf -> buf.asReadOnly() }
+
+        unpacked = Unpacked(entry, key, files)
+        unpackedCache.put(archive, entry.id, unpacked)
+        return unpacked
+    }
+}

cache/src/main/kotlin/org/openrs2/cache/Cache.kt

@@ -0,0 +1,246 @@
+package org.openrs2.cache
+
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufAllocator
+import org.openrs2.buffer.use
+import org.openrs2.crypto.XteaKey
+import org.openrs2.util.krHashCode
+import java.io.Closeable
+import java.io.FileNotFoundException
+import java.io.Flushable
+import java.nio.file.Path
+
+/**
+ * A high-level interface for reading and writing files to and from a
+ * collection of JS5 archives.
+ */
+public class Cache private constructor(
+    private val store: Store,
+    private val alloc: ByteBufAllocator,
+    unpackedCacheSize: Int
+) : Flushable, Closeable {
+    private val archives = arrayOfNulls<CacheArchive>(MAX_ARCHIVE + 1)
+    private val unpackedCache = UnpackedCache(unpackedCacheSize)
+
+    private fun init() {
+        for (archive in store.list(Js5Archive.ARCHIVESET)) {
+            val index = store.read(Js5Archive.ARCHIVESET, archive).use { compressed ->
+                Js5Compression.uncompress(compressed).use { buf ->
+                    Js5Index.read(buf)
+                }
+            }
+
+            archives[archive] = CacheArchive(alloc, index, archive, unpackedCache, store)
+        }
+    }
+
+    private fun createOrGetArchive(id: Int): Archive {
+        var archive = archives[id]
+        if (archive != null) {
+            return archive
+        }
+
+        // TODO(gpe): protocol/flags should be configurable somehow
+        val index = Js5Index(Js5Protocol.VERSIONED)
+        archive = CacheArchive(alloc, index, id, unpackedCache, store)
+        archives[id] = archive
+        return archive
+    }
+
+    // TODO(gpe): rename/move, reindex, rekey, method to go from name->id
+
+    public fun create(archive: Int) {
+        checkArchive(archive)
+        createOrGetArchive(archive)
+    }
+
+    public fun exists(archive: Int): Boolean {
+        checkArchive(archive)
+        return archives[archive] != null
+    }
+
+    public fun exists(archive: Int, group: Int): Boolean {
+        checkArchive(archive)
+        return archives[archive]?.exists(group) ?: false
+    }
+
+    public fun existsNamed(archive: Int, groupNameHash: Int): Boolean {
+        checkArchive(archive)
+        return archives[archive]?.existsNamed(groupNameHash) ?: false
+    }
+
+    public fun exists(archive: Int, group: String): Boolean {
+        return existsNamed(archive, group.krHashCode())
+    }
+
+    public fun exists(archive: Int, group: Int, file: Int): Boolean {
+        checkArchive(archive)
+        return archives[archive]?.exists(group, file) ?: false
+    }
+
+    public fun existsNamed(archive: Int, groupNameHash: Int, fileNameHash: Int): Boolean {
+        checkArchive(archive)
+        return archives[archive]?.existsNamed(groupNameHash, fileNameHash) ?: false
+    }
+
+    public fun exists(archive: Int, group: String, file: String): Boolean {
+        return existsNamed(archive, group.krHashCode(), file.krHashCode())
+    }
+
+    public fun list(): Iterator<Int> {
+        return archives.withIndex()
+            .filter { it.value != null }
+            .map { it.index }
+            .iterator()
+    }
+
+    public fun list(archive: Int): Iterator<Js5Index.Group<*>> {
+        checkArchive(archive)
+        return archives[archive]?.list() ?: throw FileNotFoundException()
+    }
+
+    public fun list(archive: Int, group: Int): Iterator<Js5Index.File> {
+        checkArchive(archive)
+        return archives[archive]?.list(group) ?: throw FileNotFoundException()
+    }
+
+    public fun listNamed(archive: Int, groupNameHash: Int): Iterator<Js5Index.File> {
+        checkArchive(archive)
+        return archives[archive]?.listNamed(groupNameHash) ?: throw FileNotFoundException()
+    }
+
+    public fun list(archive: Int, group: String): Iterator<Js5Index.File> {
+        return listNamed(archive, group.krHashCode())
+    }
+
+    public fun read(archive: Int, group: Int, file: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        checkArchive(archive)
+        return archives[archive]?.read(group, file, key) ?: throw FileNotFoundException()
+    }
+
+    public fun readNamed(archive: Int, groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        checkArchive(archive)
+        return archives[archive]?.readNamed(groupNameHash, fileNameHash, key) ?: throw FileNotFoundException()
+    }
+
+    public fun read(archive: Int, group: String, file: String, key: XteaKey = XteaKey.ZERO): ByteBuf {
+        return readNamed(archive, group.krHashCode(), file.krHashCode(), key)
+    }
+
+    public fun write(archive: Int, group: Int, file: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
+        checkArchive(archive)
+        createOrGetArchive(archive).write(group, file, buf, key)
+    }
+
+    public fun writeNamed(
+        archive: Int,
+        groupNameHash: Int,
+        fileNameHash: Int,
+        buf: ByteBuf,
+        key: XteaKey = XteaKey.ZERO
+    ) {
+        checkArchive(archive)
+        createOrGetArchive(archive).writeNamed(groupNameHash, fileNameHash, buf, key)
+    }
+
+    public fun write(archive: Int, group: String, file: String, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
+        writeNamed(archive, group.krHashCode(), file.krHashCode(), buf, key)
+    }
+
+    public fun remove(archive: Int) {
+        checkArchive(archive)
+
+        if (archives[archive] == null) {
+            return
+        }
+
+        archives[archive] = null
+
+        unpackedCache.remove(archive)
+
+        store.remove(archive)
+        store.remove(Js5Archive.ARCHIVESET, archive)
+    }
+
+    public fun remove(archive: Int, group: Int) {
+        checkArchive(archive)
+        archives[archive]?.remove(group)
+    }
+
+    public fun removeNamed(archive: Int, groupNameHash: Int) {
+        checkArchive(archive)
+        archives[archive]?.removeNamed(groupNameHash)
+    }
+
+    public fun remove(archive: Int, group: String) {
+        return removeNamed(archive, group.krHashCode())
+    }
+
+    public fun remove(archive: Int, group: Int, file: Int, key: XteaKey = XteaKey.ZERO) {
+        checkArchive(archive)
+        archives[archive]?.remove(group, file, key)
+    }
+
+    public fun removeNamed(archive: Int, groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO) {
+        checkArchive(archive)
+        archives[archive]?.removeNamed(groupNameHash, fileNameHash, key)
+    }
+
+    public fun remove(archive: Int, group: String, file: String, key: XteaKey = XteaKey.ZERO) {
+        return removeNamed(archive, group.krHashCode(), file.krHashCode(), key)
+    }
+
+    /**
+     * Writes pending changes back to the underlying [Store].
+     */
+    override fun flush() {
+        unpackedCache.flush()
+
+        for (archive in archives) {
+            archive?.flush()
+        }
+    }
+
+    /**
+     * Writes pending changes back to the underlying [Store] and clears the
+     * internal group cache.
+     */
+    public fun clear() {
+        unpackedCache.clear()
+
+        for (archive in archives) {
+            archive?.flush()
+        }
+    }
+
+    override fun close() {
+        clear()
+        store.close()
+    }
+
+    public companion object {
+        public const val MAX_ARCHIVE: Int = 254
+
+        public fun open(
+            root: Path,
+            alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
+            unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
+        ): Cache {
+            return open(Store.open(root, alloc), alloc, unpackedCacheSize)
+        }
+
+        public fun open(
+            store: Store,
+            alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
+            unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
+        ): Cache {
+            val cache = Cache(store, alloc, unpackedCacheSize)
+            cache.init()
+            return cache
+        }
+
+        private fun checkArchive(archive: Int) {
+            require(archive in 0..MAX_ARCHIVE)
+        }
+    }
+}

cache/src/main/kotlin/org/openrs2/cache/CacheArchive.kt

@@ -0,0 +1,99 @@
+package org.openrs2.cache
+
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufAllocator
+import io.netty.buffer.ByteBufUtil
+import org.openrs2.buffer.crc32
+import org.openrs2.crypto.whirlpool
+
+public class CacheArchive internal constructor(
+    alloc: ByteBufAllocator,
+    index: Js5Index,
+    archive: Int,
+    unpackedCache: UnpackedCache,
+    private val store: Store
+) : Archive(alloc, index, archive, unpackedCache) {
+    override fun packedExists(group: Int): Boolean {
+        return store.exists(archive, group)
+    }
+
+    override fun readPacked(group: Int): ByteBuf {
+        return store.read(archive, group)
+    }
+
+    override fun writePacked(group: Int, buf: ByteBuf) {
+        store.write(archive, group, buf)
+    }
+
+    override fun writePackedIndex(buf: ByteBuf) {
+        store.write(Js5Archive.ARCHIVESET, archive, buf)
+    }
+
+    override fun removePacked(group: Int) {
+        store.remove(archive, group)
+    }
+
+    override fun appendVersion(buf: ByteBuf, version: Int) {
+        buf.writeShort(version)
+    }
+
+    override fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
+        val version = VersionTrailer.strip(buf)
+        val truncatedVersion = entry.version and 0xFFFF
+        if (version != truncatedVersion) {
+            throw StoreCorruptException(
+                "Archive $archive group ${entry.id} is out of date " +
+                    "(expected version $truncatedVersion, actual version $version)"
+            )
+        }
+
+        val checksum = buf.crc32()
+        if (checksum != entry.checksum) {
+            throw StoreCorruptException(
+                "Archive $archive group ${entry.id} is corrupt " +
+                    "(expected checksum ${entry.checksum}, actual checksum $checksum)"
+            )
+        }
+
+        val length = buf.readableBytes()
+        if (index.hasLengths && length != entry.length) {
+            throw StoreCorruptException(
+                "Archive $archive group ${entry.id} is corrupt " +
+                    "(expected length ${entry.length}, actual length $length)"
+            )
+        }
+
+        if (index.hasDigests) {
+            val digest = buf.whirlpool()
+            if (!digest.contentEquals(entry.digest!!)) {
+                throw StoreCorruptException(
+                    "Archive $archive group ${entry.id} is corrupt " +
+                        "(expected digest ${ByteBufUtil.hexDump(entry.digest)}, " +
+                        "actual digest ${ByteBufUtil.hexDump(digest)})"
+                )
+            }
+        }
+    }
+
+    override fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
+        val length = buf.readableBytes()
+        if (index.hasLengths && length != entry.uncompressedLength) {
+            throw StoreCorruptException(
+                "Archive $archive group ${entry.id} is corrupt " +
+                    "(expected uncompressed length ${entry.uncompressedLength}, " +
+                    "actual length $length)"
+            )
+        }
+
+        if (index.hasUncompressedChecksums) {
+            val uncompressedChecksum = buf.crc32()
+            if (uncompressedChecksum != entry.uncompressedChecksum) {
+                throw StoreCorruptException(
+                    "Archive $archive group ${entry.id} is corrupt " +
+                        "(expected uncompressed checksum ${entry.uncompressedChecksum}, " +
+                        "actual uncompressed checksum $uncompressedChecksum)"
+                )
+            }
+        }
+    }
+}

cache/src/main/kotlin/org/openrs2/cache/Js5Pack.kt

@@ -0,0 +1,170 @@
+package org.openrs2.cache
+
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufAllocator
+import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
+import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
+import org.openrs2.buffer.use
+import java.io.Closeable
+import java.io.DataInputStream
+import java.io.FileNotFoundException
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import java.nio.file.Files
+import java.nio.file.Path
+
+/**
+ * A high-level interface for reading and writing files to and from a
+ * single JS5 archive encoded in `.js5` format.
+ */
+public class Js5Pack private constructor(
+    alloc: ByteBufAllocator,
+    index: Js5Index,
+    unpackedCacheSize: Int,
+    private var packedIndex: ByteBuf,
+    private val packed: Int2ObjectSortedMap<ByteBuf>,
+) : Archive(alloc, index, 0, UnpackedCache(unpackedCacheSize)), Closeable {
+    override fun packedExists(group: Int): Boolean {
+        return packed.containsKey(group)
+    }
+
+    override fun readPacked(group: Int): ByteBuf {
+        return packed[group]?.retainedSlice() ?: throw FileNotFoundException()
+    }
+
+    override fun writePacked(group: Int, buf: ByteBuf) {
+        packed.put(group, buf.retain().asReadOnly())?.release()
+    }
+
+    override fun writePackedIndex(buf: ByteBuf) {
+        packedIndex.release()
+        packedIndex = buf.retain().asReadOnly()
+    }
+
+    override fun removePacked(group: Int) {
+        packed.remove(group)?.release()
+    }
+
+    override fun appendVersion(buf: ByteBuf, version: Int) {
+        // empty
+    }
+
+    override fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
+        // empty
+    }
+
+    override fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
+        // empty
+    }
+
+    public fun write(path: Path) {
+        Files.newOutputStream(path).use { output ->
+            write(output)
+        }
+    }
+
+    public fun write(output: OutputStream) {
+        flush()
+
+        packedIndex.getBytes(packedIndex.readerIndex(), output, packedIndex.readableBytes())
+
+        for (compressed in packed.values) {
+            compressed.getBytes(compressed.readerIndex(), output, compressed.readableBytes())
+        }
+    }
+
+    override fun flush() {
+        unpackedCache.flush()
+        super.flush()
+    }
+
+    public fun clear() {
+        unpackedCache.clear()
+        super.flush()
+    }
+
+    override fun close() {
+        clear()
+        packedIndex.release()
+        packed.values.forEach(ByteBuf::release)
+    }
+
+    public companion object {
+        public fun create(
+            alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
+            unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
+        ): Js5Pack {
+            // TODO(gpe): protocol/flags should be configurable somehow
+            val index = Js5Index(Js5Protocol.VERSIONED)
+
+            alloc.buffer().use { uncompressed ->
+                index.write(uncompressed)
+
+                Js5Compression.compressBest(uncompressed).use { compressed ->
+                    return Js5Pack(alloc, index, unpackedCacheSize, compressed.retain(), Int2ObjectAVLTreeMap())
+                }
+            }
+        }
+
+        public fun read(
+            path: Path,
+            alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
+            unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
+        ): Js5Pack {
+            return Files.newInputStream(path).use { input ->
+                read(input, alloc, unpackedCacheSize)
+            }
+        }
+
+        public fun read(
+            input: InputStream,
+            alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
+            unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
+        ): Js5Pack {
+            val dataInput = DataInputStream(input)
+
+            readCompressed(dataInput, alloc).use { compressed ->
+                val index = Js5Compression.uncompress(compressed.slice()).use { uncompressed ->
+                    Js5Index.read(uncompressed)
+                }
+
+                val packed = Int2ObjectAVLTreeMap<ByteBuf>()
+                try {
+                    for (group in index) {
+                        packed[group.id] = readCompressed(dataInput, alloc).asReadOnly()
+                    }
+
+                    packed.values.forEach(ByteBuf::retain)
+                    return Js5Pack(alloc, index, unpackedCacheSize, compressed.retain(), packed)
+                } finally {
+                    packed.values.forEach(ByteBuf::release)
+                }
+            }
+        }
+
+        private fun readCompressed(input: DataInputStream, alloc: ByteBufAllocator): ByteBuf {
+            val typeId = input.readUnsignedByte()
+            val type = Js5CompressionType.fromOrdinal(typeId)
+                ?: throw IOException("Invalid compression type: $typeId")
+
+            val len = input.readInt()
+            if (len < 0) {
+                throw IOException("Length is negative: $len")
+            }
+
+            val lenWithUncompressedLen = if (type == Js5CompressionType.UNCOMPRESSED) {
+                len
+            } else {
+                len + 4
+            }
+
+            alloc.buffer(lenWithUncompressedLen + 5, lenWithUncompressedLen + 5).use { buf ->
+                buf.writeByte(typeId)
+                buf.writeInt(len)
+                buf.writeBytes(input, lenWithUncompressedLen)
+                return buf.retain()
+            }
+        }
+    }
+}

cache/src/main/kotlin/org/openrs2/cache/UnpackedCache.kt

@@ -0,0 +1,75 @@
+package org.openrs2.cache
+
+import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap
+import it.unimi.dsi.fastutil.longs.LongArrayList
+
+internal class UnpackedCache(
+    private val capacity: Int
+) {
+    private val cache = Long2ObjectLinkedOpenHashMap<Archive.Unpacked>()
+
+    init {
+        require(capacity >= 1)
+    }
+
+    fun get(archive: Int, group: Int): Archive.Unpacked? {
+        return cache.getAndMoveToLast(key(archive, group))
+    }
+
+    fun put(archive: Int, group: Int, unpacked: Archive.Unpacked) {
+        while (cache.size >= capacity) {
+            val lru = cache.removeFirst()
+            lru.flush()
+            lru.release()
+        }
+
+        cache.putAndMoveToLast(key(archive, group), unpacked)?.release()
+    }
+
+    fun remove(archive: Int) {
+        val start = key(archive, 0)
+        val end = key(archive + 1, 0)
+
+        val keys = LongArrayList()
+
+        val it = cache.keys.iterator(start)
+        while (it.hasNext()) {
+            val key = it.nextLong()
+            if (key >= end) {
+                break
+            }
+            keys += key
+        }
+
+        for (i in 0 until keys.size) {
+            cache.remove(keys.getLong(i)).release()
+        }
+    }
+
+    fun remove(archive: Int, group: Int) {
+        cache.remove(key(archive, group))?.release()
+    }
+
+    fun flush() {
+        for (unpacked in cache.values) {
+            unpacked.flush()
+        }
+    }
+
+    fun clear() {
+        for (unpacked in cache.values) {
+            unpacked.flush()
+            unpacked.release()
+        }
+
+        cache.clear()
+    }
+
+    private fun key(archive: Int, group: Int): Long {
+        return (archive.toLong() shl 32) or group.toLong()
+    }
+
+    companion object {
+        const val DEFAULT_CAPACITY: Int = 1024
+    }
+}