Add initial high-level cache API

It supports reading and writing a cache backed by a Store, as well as
standalone .js5 files.

I'm not quite as happy with this as I am with the lower-level API yet,
and there are still a few remaining TODOs.

Signed-off-by: Graham <gpe@openrs2.org>
bzip2
Graham 4 years ago
parent 6f13a0a737
commit ad0cdb6056
  1. 347
      cache/src/main/kotlin/org/openrs2/cache/Archive.kt
  2. 246
      cache/src/main/kotlin/org/openrs2/cache/Cache.kt
  3. 99
      cache/src/main/kotlin/org/openrs2/cache/CacheArchive.kt
  4. 170
      cache/src/main/kotlin/org/openrs2/cache/Js5Pack.kt
  5. 75
      cache/src/main/kotlin/org/openrs2/cache/UnpackedCache.kt

@ -0,0 +1,347 @@
package org.openrs2.cache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMaps
import org.openrs2.buffer.crc32
import org.openrs2.buffer.use
import org.openrs2.crypto.XteaKey
import org.openrs2.crypto.whirlpool
import org.openrs2.util.krHashCode
import java.io.FileNotFoundException
import java.io.Flushable
public abstract class Archive internal constructor(
protected val alloc: ByteBufAllocator,
protected val index: Js5Index,
protected val archive: Int,
internal val unpackedCache: UnpackedCache
) : Flushable {
private var dirty = false
internal inner class Unpacked(
private val entry: Js5Index.MutableGroup,
val key: XteaKey,
private var files: Int2ObjectSortedMap<ByteBuf>
) {
private var dirty = false
private fun ensureWritable() {
if (files.size == 1 && files is Int2ObjectSortedMaps.Singleton) {
files = Int2ObjectAVLTreeMap(files)
}
}
fun read(file: Int): ByteBuf {
val fileEntry = entry[file] ?: throw FileNotFoundException()
return files[fileEntry.id]!!.retainedSlice()
}
fun readNamed(fileNameHash: Int): ByteBuf {
val fileEntry = entry.getNamed(fileNameHash) ?: throw FileNotFoundException()
return files[fileEntry.id]!!.retainedSlice()
}
fun write(file: Int, buf: ByteBuf) {
ensureWritable()
val fileEntry = entry.createOrGet(file)
files.put(fileEntry.id, buf.copy().asReadOnly())?.release()
dirty = true
}
fun writeNamed(fileNameHash: Int, buf: ByteBuf) {
ensureWritable()
val fileEntry = entry.createOrGetNamed(fileNameHash)
files.put(fileEntry.id, buf.copy().asReadOnly())?.release()
dirty = true
}
fun remove(file: Int) {
ensureWritable()
val fileEntry = entry.remove(file) ?: return
files.remove(fileEntry.id)?.release()
dirty = true
}
fun removeNamed(fileNameHash: Int) {
ensureWritable()
val fileEntry = entry.removeNamed(fileNameHash) ?: return
files.remove(fileEntry.id)?.release()
dirty = true
}
fun flush() {
if (!dirty) {
return
}
Group.pack(files).use { buf ->
if (index.hasLengths) {
entry.uncompressedLength = buf.readableBytes()
}
if (index.hasUncompressedChecksums) {
entry.uncompressedChecksum = buf.crc32()
}
Js5Compression.compressBest(buf, key = key).use { compressed ->
entry.checksum = compressed.crc32()
if (index.hasLengths) {
entry.length = compressed.readableBytes()
}
if (index.hasDigests) {
entry.digest = compressed.whirlpool()
}
appendVersion(buf, ++entry.version)
writePacked(entry.id, compressed)
}
}
dirty = false
}
fun release() {
files.values.forEach(ByteBuf::release)
}
}
// TODO(gpe): rename/move, reindex, rekey, method to go from name->id
public fun exists(group: Int): Boolean {
require(group >= 0)
return index.contains(group)
}
public fun existsNamed(groupNameHash: Int): Boolean {
return index.containsNamed(groupNameHash)
}
public fun exists(group: String): Boolean {
return existsNamed(group.krHashCode())
}
public fun exists(group: Int, file: Int): Boolean {
require(group >= 0 && file >= 0)
val entry = index[group] ?: return false
return entry.contains(file)
}
public fun existsNamed(groupNameHash: Int, fileNameHash: Int): Boolean {
val entry = index.getNamed(groupNameHash) ?: return false
return entry.containsNamed(fileNameHash)
}
public fun exists(group: String, file: String): Boolean {
return existsNamed(group.krHashCode(), file.krHashCode())
}
public fun list(): Iterator<Js5Index.Group<*>> {
return index.iterator()
}
public fun list(group: Int): Iterator<Js5Index.File> {
require(group >= 0)
val entry = index[group] ?: throw FileNotFoundException()
return entry.iterator()
}
public fun listNamed(groupNameHash: Int): Iterator<Js5Index.File> {
val entry = index.getNamed(groupNameHash) ?: throw FileNotFoundException()
return entry.iterator()
}
public fun list(group: String): Iterator<Js5Index.File> {
return listNamed(group.krHashCode())
}
public fun read(group: Int, file: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
require(group >= 0 && file >= 0)
val entry = index[group] ?: throw FileNotFoundException()
val unpacked = getUnpacked(entry, key)
return unpacked.read(file)
}
public fun readNamed(groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
val entry = index.getNamed(groupNameHash) ?: throw FileNotFoundException()
val unpacked = getUnpacked(entry, key)
return unpacked.readNamed(fileNameHash)
}
public fun read(group: String, file: String, key: XteaKey = XteaKey.ZERO): ByteBuf {
return readNamed(group.krHashCode(), file.krHashCode(), key)
}
public fun write(group: Int, file: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
require(group >= 0 && file >= 0)
val entry = index.createOrGet(group)
val unpacked = createOrGetUnpacked(entry, key, isOverwriting(entry, file))
unpacked.write(file, buf)
dirty = true
}
public fun writeNamed(groupNameHash: Int, fileNameHash: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
val entry = index.createOrGetNamed(groupNameHash)
val unpacked = createOrGetUnpacked(entry, key, isOverwritingNamed(entry, fileNameHash))
unpacked.writeNamed(fileNameHash, buf)
dirty = true
index.hasNames = true
}
public fun write(group: String, file: String, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
return writeNamed(group.krHashCode(), file.krHashCode(), buf, key)
}
public fun remove(group: Int) {
require(group >= 0)
val entry = index.remove(group) ?: return
unpackedCache.remove(archive, entry.id)
removePacked(entry.id)
dirty = true
}
public fun removeNamed(groupNameHash: Int) {
val entry = index.removeNamed(groupNameHash) ?: return
unpackedCache.remove(archive, entry.id)
removePacked(entry.id)
dirty = true
}
public fun remove(group: String) {
return removeNamed(group.krHashCode())
}
public fun remove(group: Int, file: Int, key: XteaKey = XteaKey.ZERO) {
require(group >= 0 && file >= 0)
val entry = index[group] ?: return
if (isOverwriting(entry, file)) {
remove(group)
return
}
val unpacked = getUnpacked(entry, key)
unpacked.remove(file)
dirty = true
}
public fun removeNamed(groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO) {
val entry = index.getNamed(groupNameHash) ?: return
if (isOverwritingNamed(entry, fileNameHash)) {
removeNamed(groupNameHash)
return
}
val unpacked = getUnpacked(entry, key)
unpacked.removeNamed(fileNameHash)
dirty = true
}
public fun remove(group: String, file: String, key: XteaKey = XteaKey.ZERO) {
return removeNamed(group.krHashCode(), file.krHashCode(), key)
}
public override fun flush() {
if (!dirty) {
return
}
index.version++
alloc.buffer().use { buf ->
index.write(buf)
Js5Compression.compressBest(buf).use { compressed ->
writePackedIndex(compressed)
}
}
dirty = false
}
protected abstract fun packedExists(group: Int): Boolean
protected abstract fun readPacked(group: Int): ByteBuf
protected abstract fun writePacked(group: Int, buf: ByteBuf)
protected abstract fun writePackedIndex(buf: ByteBuf)
protected abstract fun removePacked(group: Int)
protected abstract fun appendVersion(buf: ByteBuf, version: Int)
protected abstract fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup)
protected abstract fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup)
private fun isOverwriting(entry: Js5Index.MutableGroup, file: Int): Boolean {
val fileEntry = entry.singleOrNull() ?: return false
return fileEntry.id == file
}
private fun isOverwritingNamed(entry: Js5Index.MutableGroup, fileNameHash: Int): Boolean {
val fileEntry = entry.singleOrNull() ?: return false
return fileEntry.nameHash == fileNameHash
}
private fun createOrGetUnpacked(entry: Js5Index.MutableGroup, key: XteaKey, overwrite: Boolean): Unpacked {
return if (entry.size == 0 || overwrite) {
val unpacked = Unpacked(entry, key, Int2ObjectAVLTreeMap())
unpackedCache.put(archive, entry.id, unpacked)
return unpacked
} else {
getUnpacked(entry, key)
}
}
private fun getUnpacked(entry: Js5Index.MutableGroup, key: XteaKey): Unpacked {
var unpacked = unpackedCache.get(archive, entry.id)
if (unpacked != null) {
/*
* If we've already unpacked the group, we check the programmer
* is using the correct key to ensure the code always works,
* regardless of group cache size/invalidation behaviour.
*/
require(unpacked.key == key) {
"Invalid key for archive $archive group ${entry.id} (expected ${unpacked!!.key}, actual $key)"
}
return unpacked
}
if (!packedExists(entry.id)) {
throw StoreCorruptException("Archive $archive group ${entry.id} is missing")
}
val files = readPacked(entry.id).use { compressed ->
// TODO(gpe): check for trailing data?
verifyCompressed(compressed, entry)
Js5Compression.uncompress(compressed, key).use { buf ->
verifyUncompressed(buf, entry)
Group.unpack(buf, entry)
}
}
files.replaceAll { _, buf -> buf.asReadOnly() }
unpacked = Unpacked(entry, key, files)
unpackedCache.put(archive, entry.id, unpacked)
return unpacked
}
}

@ -0,0 +1,246 @@
package org.openrs2.cache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import org.openrs2.buffer.use
import org.openrs2.crypto.XteaKey
import org.openrs2.util.krHashCode
import java.io.Closeable
import java.io.FileNotFoundException
import java.io.Flushable
import java.nio.file.Path
/**
* A high-level interface for reading and writing files to and from a
* collection of JS5 archives.
*/
public class Cache private constructor(
private val store: Store,
private val alloc: ByteBufAllocator,
unpackedCacheSize: Int
) : Flushable, Closeable {
private val archives = arrayOfNulls<CacheArchive>(MAX_ARCHIVE + 1)
private val unpackedCache = UnpackedCache(unpackedCacheSize)
private fun init() {
for (archive in store.list(Js5Archive.ARCHIVESET)) {
val index = store.read(Js5Archive.ARCHIVESET, archive).use { compressed ->
Js5Compression.uncompress(compressed).use { buf ->
Js5Index.read(buf)
}
}
archives[archive] = CacheArchive(alloc, index, archive, unpackedCache, store)
}
}
private fun createOrGetArchive(id: Int): Archive {
var archive = archives[id]
if (archive != null) {
return archive
}
// TODO(gpe): protocol/flags should be configurable somehow
val index = Js5Index(Js5Protocol.VERSIONED)
archive = CacheArchive(alloc, index, id, unpackedCache, store)
archives[id] = archive
return archive
}
// TODO(gpe): rename/move, reindex, rekey, method to go from name->id
public fun create(archive: Int) {
checkArchive(archive)
createOrGetArchive(archive)
}
public fun exists(archive: Int): Boolean {
checkArchive(archive)
return archives[archive] != null
}
public fun exists(archive: Int, group: Int): Boolean {
checkArchive(archive)
return archives[archive]?.exists(group) ?: false
}
public fun existsNamed(archive: Int, groupNameHash: Int): Boolean {
checkArchive(archive)
return archives[archive]?.existsNamed(groupNameHash) ?: false
}
public fun exists(archive: Int, group: String): Boolean {
return existsNamed(archive, group.krHashCode())
}
public fun exists(archive: Int, group: Int, file: Int): Boolean {
checkArchive(archive)
return archives[archive]?.exists(group, file) ?: false
}
public fun existsNamed(archive: Int, groupNameHash: Int, fileNameHash: Int): Boolean {
checkArchive(archive)
return archives[archive]?.existsNamed(groupNameHash, fileNameHash) ?: false
}
public fun exists(archive: Int, group: String, file: String): Boolean {
return existsNamed(archive, group.krHashCode(), file.krHashCode())
}
public fun list(): Iterator<Int> {
return archives.withIndex()
.filter { it.value != null }
.map { it.index }
.iterator()
}
public fun list(archive: Int): Iterator<Js5Index.Group<*>> {
checkArchive(archive)
return archives[archive]?.list() ?: throw FileNotFoundException()
}
public fun list(archive: Int, group: Int): Iterator<Js5Index.File> {
checkArchive(archive)
return archives[archive]?.list(group) ?: throw FileNotFoundException()
}
public fun listNamed(archive: Int, groupNameHash: Int): Iterator<Js5Index.File> {
checkArchive(archive)
return archives[archive]?.listNamed(groupNameHash) ?: throw FileNotFoundException()
}
public fun list(archive: Int, group: String): Iterator<Js5Index.File> {
return listNamed(archive, group.krHashCode())
}
public fun read(archive: Int, group: Int, file: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
checkArchive(archive)
return archives[archive]?.read(group, file, key) ?: throw FileNotFoundException()
}
public fun readNamed(archive: Int, groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO): ByteBuf {
checkArchive(archive)
return archives[archive]?.readNamed(groupNameHash, fileNameHash, key) ?: throw FileNotFoundException()
}
public fun read(archive: Int, group: String, file: String, key: XteaKey = XteaKey.ZERO): ByteBuf {
return readNamed(archive, group.krHashCode(), file.krHashCode(), key)
}
public fun write(archive: Int, group: Int, file: Int, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
checkArchive(archive)
createOrGetArchive(archive).write(group, file, buf, key)
}
public fun writeNamed(
archive: Int,
groupNameHash: Int,
fileNameHash: Int,
buf: ByteBuf,
key: XteaKey = XteaKey.ZERO
) {
checkArchive(archive)
createOrGetArchive(archive).writeNamed(groupNameHash, fileNameHash, buf, key)
}
public fun write(archive: Int, group: String, file: String, buf: ByteBuf, key: XteaKey = XteaKey.ZERO) {
writeNamed(archive, group.krHashCode(), file.krHashCode(), buf, key)
}
public fun remove(archive: Int) {
checkArchive(archive)
if (archives[archive] == null) {
return
}
archives[archive] = null
unpackedCache.remove(archive)
store.remove(archive)
store.remove(Js5Archive.ARCHIVESET, archive)
}
public fun remove(archive: Int, group: Int) {
checkArchive(archive)
archives[archive]?.remove(group)
}
public fun removeNamed(archive: Int, groupNameHash: Int) {
checkArchive(archive)
archives[archive]?.removeNamed(groupNameHash)
}
public fun remove(archive: Int, group: String) {
return removeNamed(archive, group.krHashCode())
}
public fun remove(archive: Int, group: Int, file: Int, key: XteaKey = XteaKey.ZERO) {
checkArchive(archive)
archives[archive]?.remove(group, file, key)
}
public fun removeNamed(archive: Int, groupNameHash: Int, fileNameHash: Int, key: XteaKey = XteaKey.ZERO) {
checkArchive(archive)
archives[archive]?.removeNamed(groupNameHash, fileNameHash, key)
}
public fun remove(archive: Int, group: String, file: String, key: XteaKey = XteaKey.ZERO) {
return removeNamed(archive, group.krHashCode(), file.krHashCode(), key)
}
/**
* Writes pending changes back to the underlying [Store].
*/
override fun flush() {
unpackedCache.flush()
for (archive in archives) {
archive?.flush()
}
}
/**
* Writes pending changes back to the underlying [Store] and clears the
* internal group cache.
*/
public fun clear() {
unpackedCache.clear()
for (archive in archives) {
archive?.flush()
}
}
override fun close() {
clear()
store.close()
}
public companion object {
public const val MAX_ARCHIVE: Int = 254
public fun open(
root: Path,
alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
): Cache {
return open(Store.open(root, alloc), alloc, unpackedCacheSize)
}
public fun open(
store: Store,
alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
): Cache {
val cache = Cache(store, alloc, unpackedCacheSize)
cache.init()
return cache
}
private fun checkArchive(archive: Int) {
require(archive in 0..MAX_ARCHIVE)
}
}
}

@ -0,0 +1,99 @@
package org.openrs2.cache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.ByteBufUtil
import org.openrs2.buffer.crc32
import org.openrs2.crypto.whirlpool
public class CacheArchive internal constructor(
alloc: ByteBufAllocator,
index: Js5Index,
archive: Int,
unpackedCache: UnpackedCache,
private val store: Store
) : Archive(alloc, index, archive, unpackedCache) {
override fun packedExists(group: Int): Boolean {
return store.exists(archive, group)
}
override fun readPacked(group: Int): ByteBuf {
return store.read(archive, group)
}
override fun writePacked(group: Int, buf: ByteBuf) {
store.write(archive, group, buf)
}
override fun writePackedIndex(buf: ByteBuf) {
store.write(Js5Archive.ARCHIVESET, archive, buf)
}
override fun removePacked(group: Int) {
store.remove(archive, group)
}
override fun appendVersion(buf: ByteBuf, version: Int) {
buf.writeShort(version)
}
override fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
val version = VersionTrailer.strip(buf)
val truncatedVersion = entry.version and 0xFFFF
if (version != truncatedVersion) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is out of date " +
"(expected version $truncatedVersion, actual version $version)"
)
}
val checksum = buf.crc32()
if (checksum != entry.checksum) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is corrupt " +
"(expected checksum ${entry.checksum}, actual checksum $checksum)"
)
}
val length = buf.readableBytes()
if (index.hasLengths && length != entry.length) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is corrupt " +
"(expected length ${entry.length}, actual length $length)"
)
}
if (index.hasDigests) {
val digest = buf.whirlpool()
if (!digest.contentEquals(entry.digest!!)) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is corrupt " +
"(expected digest ${ByteBufUtil.hexDump(entry.digest)}, " +
"actual digest ${ByteBufUtil.hexDump(digest)})"
)
}
}
}
override fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
val length = buf.readableBytes()
if (index.hasLengths && length != entry.uncompressedLength) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is corrupt " +
"(expected uncompressed length ${entry.uncompressedLength}, " +
"actual length $length)"
)
}
if (index.hasUncompressedChecksums) {
val uncompressedChecksum = buf.crc32()
if (uncompressedChecksum != entry.uncompressedChecksum) {
throw StoreCorruptException(
"Archive $archive group ${entry.id} is corrupt " +
"(expected uncompressed checksum ${entry.uncompressedChecksum}, " +
"actual uncompressed checksum $uncompressedChecksum)"
)
}
}
}
}

@ -0,0 +1,170 @@
package org.openrs2.cache
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
import org.openrs2.buffer.use
import java.io.Closeable
import java.io.DataInputStream
import java.io.FileNotFoundException
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.nio.file.Files
import java.nio.file.Path
/**
* A high-level interface for reading and writing files to and from a
* single JS5 archive encoded in `.js5` format.
*/
public class Js5Pack private constructor(
alloc: ByteBufAllocator,
index: Js5Index,
unpackedCacheSize: Int,
private var packedIndex: ByteBuf,
private val packed: Int2ObjectSortedMap<ByteBuf>,
) : Archive(alloc, index, 0, UnpackedCache(unpackedCacheSize)), Closeable {
override fun packedExists(group: Int): Boolean {
return packed.containsKey(group)
}
override fun readPacked(group: Int): ByteBuf {
return packed[group]?.retainedSlice() ?: throw FileNotFoundException()
}
override fun writePacked(group: Int, buf: ByteBuf) {
packed.put(group, buf.retain().asReadOnly())?.release()
}
override fun writePackedIndex(buf: ByteBuf) {
packedIndex.release()
packedIndex = buf.retain().asReadOnly()
}
override fun removePacked(group: Int) {
packed.remove(group)?.release()
}
override fun appendVersion(buf: ByteBuf, version: Int) {
// empty
}
override fun verifyCompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
// empty
}
override fun verifyUncompressed(buf: ByteBuf, entry: Js5Index.MutableGroup) {
// empty
}
public fun write(path: Path) {
Files.newOutputStream(path).use { output ->
write(output)
}
}
public fun write(output: OutputStream) {
flush()
packedIndex.getBytes(packedIndex.readerIndex(), output, packedIndex.readableBytes())
for (compressed in packed.values) {
compressed.getBytes(compressed.readerIndex(), output, compressed.readableBytes())
}
}
override fun flush() {
unpackedCache.flush()
super.flush()
}
public fun clear() {
unpackedCache.clear()
super.flush()
}
override fun close() {
clear()
packedIndex.release()
packed.values.forEach(ByteBuf::release)
}
public companion object {
public fun create(
alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
): Js5Pack {
// TODO(gpe): protocol/flags should be configurable somehow
val index = Js5Index(Js5Protocol.VERSIONED)
alloc.buffer().use { uncompressed ->
index.write(uncompressed)
Js5Compression.compressBest(uncompressed).use { compressed ->
return Js5Pack(alloc, index, unpackedCacheSize, compressed.retain(), Int2ObjectAVLTreeMap())
}
}
}
public fun read(
path: Path,
alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
): Js5Pack {
return Files.newInputStream(path).use { input ->
read(input, alloc, unpackedCacheSize)
}
}
public fun read(
input: InputStream,
alloc: ByteBufAllocator = ByteBufAllocator.DEFAULT,
unpackedCacheSize: Int = UnpackedCache.DEFAULT_CAPACITY
): Js5Pack {
val dataInput = DataInputStream(input)
readCompressed(dataInput, alloc).use { compressed ->
val index = Js5Compression.uncompress(compressed.slice()).use { uncompressed ->
Js5Index.read(uncompressed)
}
val packed = Int2ObjectAVLTreeMap<ByteBuf>()
try {
for (group in index) {
packed[group.id] = readCompressed(dataInput, alloc).asReadOnly()
}
packed.values.forEach(ByteBuf::retain)
return Js5Pack(alloc, index, unpackedCacheSize, compressed.retain(), packed)
} finally {
packed.values.forEach(ByteBuf::release)
}
}
}
private fun readCompressed(input: DataInputStream, alloc: ByteBufAllocator): ByteBuf {
val typeId = input.readUnsignedByte()
val type = Js5CompressionType.fromOrdinal(typeId)
?: throw IOException("Invalid compression type: $typeId")
val len = input.readInt()
if (len < 0) {
throw IOException("Length is negative: $len")
}
val lenWithUncompressedLen = if (type == Js5CompressionType.UNCOMPRESSED) {
len
} else {
len + 4
}
alloc.buffer(lenWithUncompressedLen + 5, lenWithUncompressedLen + 5).use { buf ->
buf.writeByte(typeId)
buf.writeInt(len)
buf.writeBytes(input, lenWithUncompressedLen)
return buf.retain()
}
}
}
}

@ -0,0 +1,75 @@
package org.openrs2.cache
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap
import it.unimi.dsi.fastutil.longs.LongArrayList
internal class UnpackedCache(
private val capacity: Int
) {
private val cache = Long2ObjectLinkedOpenHashMap<Archive.Unpacked>()
init {
require(capacity >= 1)
}
fun get(archive: Int, group: Int): Archive.Unpacked? {
return cache.getAndMoveToLast(key(archive, group))
}
fun put(archive: Int, group: Int, unpacked: Archive.Unpacked) {
while (cache.size >= capacity) {
val lru = cache.removeFirst()
lru.flush()
lru.release()
}
cache.putAndMoveToLast(key(archive, group), unpacked)?.release()
}
fun remove(archive: Int) {
val start = key(archive, 0)
val end = key(archive + 1, 0)
val keys = LongArrayList()
val it = cache.keys.iterator(start)
while (it.hasNext()) {
val key = it.nextLong()
if (key >= end) {
break
}
keys += key
}
for (i in 0 until keys.size) {
cache.remove(keys.getLong(i)).release()
}
}
fun remove(archive: Int, group: Int) {
cache.remove(key(archive, group))?.release()
}
fun flush() {
for (unpacked in cache.values) {
unpacked.flush()
}
}
fun clear() {
for (unpacked in cache.values) {
unpacked.flush()
unpacked.release()
}
cache.clear()
}
private fun key(archive: Int, group: Int): Long {
return (archive.toLong() shl 32) or group.toLong()
}
companion object {
const val DEFAULT_CAPACITY: Int = 1024
}
}
Loading…
Cancel
Save