Add read-only Js5Index.{Group,File} types

These will be used by the high-level cache API, where we don't want to
expose mutable versions of the group/file types as that would allow the
index/cache to get out of sync.

Signed-off-by: Graham <gpe@openrs2.org>
bzip2
Graham 4 years ago
parent e0d29e5ac2
commit ce741279b4
  1. 2
      archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt
  2. 2
      cache/src/main/kotlin/org/openrs2/cache/Group.kt
  3. 42
      cache/src/main/kotlin/org/openrs2/cache/Js5Index.kt
  4. 2
      cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt
  5. 17
      cache/src/main/kotlin/org/openrs2/cache/MutableNamedEntry.kt
  6. 371
      cache/src/main/kotlin/org/openrs2/cache/MutableNamedEntryCollection.kt
  7. 14
      cache/src/main/kotlin/org/openrs2/cache/NamedEntry.kt
  8. 366
      cache/src/main/kotlin/org/openrs2/cache/NamedEntryCollection.kt
  9. 8
      cache/src/test/kotlin/org/openrs2/cache/NamedEntryCollectionTest.kt

@ -277,7 +277,7 @@ public class MapRenderer @Inject constructor(
connection: Connection, connection: Connection,
masterIndexId: Int, masterIndexId: Int,
archiveId: Int, archiveId: Int,
group: Js5Index.Group group: Js5Index.Group<*>
): Int2ObjectSortedMap<ByteBuf>? { ): Int2ObjectSortedMap<ByteBuf>? {
connection.prepareStatement( connection.prepareStatement(
""" """

@ -6,7 +6,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
import org.openrs2.buffer.use import org.openrs2.buffer.use
public object Group { public object Group {
public fun unpack(input: ByteBuf, group: Js5Index.Group): Int2ObjectSortedMap<ByteBuf> { public fun unpack(input: ByteBuf, group: Js5Index.Group<*>): Int2ObjectSortedMap<ByteBuf> {
require(group.size >= 1) require(group.size >= 1)
val singleEntry = group.singleOrNull() val singleEntry = group.singleOrNull()

@ -13,17 +13,25 @@ public class Js5Index(
public var hasDigests: Boolean = false, public var hasDigests: Boolean = false,
public var hasLengths: Boolean = false, public var hasLengths: Boolean = false,
public var hasUncompressedChecksums: Boolean = false public var hasUncompressedChecksums: Boolean = false
) : NamedEntryCollection<Js5Index.Group>(::Group) { ) : MutableNamedEntryCollection<Js5Index.MutableGroup>(::MutableGroup) {
public class Group internal constructor( public interface Group<out T : File> : NamedEntryCollection<T>, NamedEntry {
parent: NamedEntryCollection<Group>, public val version: Int
public val checksum: Int
public val uncompressedChecksum: Int
public val length: Int
public val uncompressedLength: Int
}
public class MutableGroup internal constructor(
parent: MutableNamedEntryCollection<MutableGroup>,
override val id: Int override val id: Int
) : NamedEntryCollection<File>(::File), NamedEntry { ) : MutableNamedEntryCollection<MutableFile>(::MutableFile), MutableNamedEntry, Group<MutableFile> {
private var parent: NamedEntryCollection<Group>? = parent private var parent: MutableNamedEntryCollection<MutableGroup>? = parent
public var version: Int = 0 public override var version: Int = 0
public var checksum: Int = 0 public override var checksum: Int = 0
public var uncompressedChecksum: Int = 0 public override var uncompressedChecksum: Int = 0
public var length: Int = 0 public override var length: Int = 0
public var uncompressedLength: Int = 0 public override var uncompressedLength: Int = 0
public override var nameHash: Int = -1 public override var nameHash: Int = -1
set(value) { set(value) {
@ -47,7 +55,7 @@ public class Js5Index(
if (javaClass != other?.javaClass) return false if (javaClass != other?.javaClass) return false
if (!super.equals(other)) return false if (!super.equals(other)) return false
other as Group other as MutableGroup
if (id != other.id) return false if (id != other.id) return false
if (version != other.version) return false if (version != other.version) return false
@ -90,11 +98,13 @@ public class Js5Index(
} }
} }
public class File internal constructor( public interface File : NamedEntry
parent: NamedEntryCollection<File>,
public class MutableFile internal constructor(
parent: MutableNamedEntryCollection<MutableFile>,
override val id: Int, override val id: Int,
) : NamedEntry { ) : MutableNamedEntry, File {
private var parent: NamedEntryCollection<File>? = parent private var parent: MutableNamedEntryCollection<MutableFile>? = parent
public override var nameHash: Int = -1 public override var nameHash: Int = -1
set(value) { set(value) {
@ -111,7 +121,7 @@ public class Js5Index(
if (this === other) return true if (this === other) return true
if (javaClass != other?.javaClass) return false if (javaClass != other?.javaClass) return false
other as File other as MutableFile
if (id != other.id) return false if (id != other.id) return false
if (nameHash != other.nameHash) return false if (nameHash != other.nameHash) return false

@ -140,7 +140,7 @@ public data class Js5MasterIndex(
val version = index.version val version = index.version
val groups = index.size val groups = index.size
val totalUncompressedLength = index.sumOf(Js5Index.Group::uncompressedLength) val totalUncompressedLength = index.sumOf(Js5Index.Group<*>::uncompressedLength)
// TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above? // TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above?
Entry(version, checksum, groups, totalUncompressedLength, digest) Entry(version, checksum, groups, totalUncompressedLength, digest)

@ -0,0 +1,17 @@
package org.openrs2.cache
import org.openrs2.util.krHashCode
public interface MutableNamedEntry : NamedEntry {
public override var nameHash: Int
public fun setName(name: String) {
nameHash = name.krHashCode()
}
public fun clearName() {
nameHash = -1
}
public fun remove()
}

@ -0,0 +1,371 @@
package org.openrs2.cache
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import it.unimi.dsi.fastutil.ints.Int2ObjectMap
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap
import it.unimi.dsi.fastutil.ints.IntAVLTreeSet
import it.unimi.dsi.fastutil.ints.IntSortedSet
import it.unimi.dsi.fastutil.ints.IntSortedSets
import org.openrs2.util.krHashCode
/**
* A specialist collection type entirely designed for use by the [Js5Index]
* class.
*
* Entries may be accessed by their ID number or by the hash of their name, if
* they are named.
*
* Entries are sorted by their ID. The IDs should be fairly dense starting
* around 0, as the client stores entries in an array. Nevertheless, this
* implementation currently supports sparse IDs efficiently as the entries are
* actually stored in a tree map, primarily to make manipulation of the
* collection simpler (the client's implementation has the luxury of being
* read-only!)
*
* As the vast majority of [Js5Index.Group]s only have a single [Js5Index.File]
* entry, there is a special case for single entry collections. This avoids the
* memory overhead of the underlying collections in this special case, at the
* expense of making the logic slightly more complicated.
*
* Entries without a name have a name hash of -1. This is consistent with the
* client, which sets the name hash array to -1 before populating it. If the
* index or group is sparse, any non-existent IDs still have a hash of -1, which
* is passed into the client's IntHashTable class. This makes -1 unusable, so I
* feel this choice is justified.
*
* In practice, I think indexes generally either have names for every file or
* none at all, but allowing a mixture in this implementation makes mutating
* the index simpler.
*
* This implementation permits multiple entries to have the same name hash. In
* the event of a collision, the colliding entry with the lowest ID is returned
* from methods that look up an entry by its name hash. This implementation
* still behaves correctly if one of the colliding entries is removed: it will
* always return the entry with the next lowest ID.
*
* This is an edge case that is unlikely to be hit in practice: the 550 cache
* has no collisions, and the 876 cache only has a single collision (the hash
* for the empty string).
*/
public abstract class MutableNamedEntryCollection<T : MutableNamedEntry>(
private val entryConstructor: (MutableNamedEntryCollection<T>, Int) -> T
) : NamedEntryCollection<T>, MutableIterable<T> {
private var singleEntry: T? = null
private var entries: Int2ObjectAVLTreeMap<T>? = null
// XXX(gpe): unfortunately fastutil doesn't have a multimap type
private var nameHashTable: Int2ObjectMap<IntSortedSet>? = null
public override val size: Int
get() {
if (singleEntry != null) {
return 1
}
val entries = entries ?: return 0
return entries.size
}
public override val capacity: Int
get() {
val entry = singleEntry
if (entry != null) {
return entry.id + 1
}
val entries = entries ?: return 0
assert(entries.isNotEmpty())
return entries.lastIntKey() + 1
}
public override operator fun contains(id: Int): Boolean {
require(id >= 0)
val entry = singleEntry
if (entry?.id == id) {
return true
}
val entries = entries ?: return false
return entries.containsKey(id)
}
public override fun containsNamed(nameHash: Int): Boolean {
require(nameHash != -1)
val entry = singleEntry
if (entry?.nameHash == nameHash) {
return true
}
return nameHashTable?.containsKey(nameHash) ?: false
}
public override operator fun get(id: Int): T? {
require(id >= 0)
val entry = singleEntry
if (entry?.id == id) {
return entry
}
val entries = entries ?: return null
return entries[id]
}
public override fun getNamed(nameHash: Int): T? {
require(nameHash != -1)
val entry = singleEntry
if (entry?.nameHash == nameHash) {
return entry
}
val nameHashTable = nameHashTable ?: return null
val ids = nameHashTable.getOrDefault(nameHash, IntSortedSets.EMPTY_SET)
return if (ids.isNotEmpty()) {
get(ids.firstInt())!!
} else {
null
}
}
public fun createOrGet(id: Int): T {
var entry = get(id)
if (entry != null) {
return entry
}
entry = entryConstructor(this, id)
val singleEntry = singleEntry
var entries = entries
if (singleEntry == null && entries == null) {
this.singleEntry = entry
return entry
}
if (entries == null) {
entries = Int2ObjectAVLTreeMap<T>()
if (singleEntry != null) {
entries[singleEntry.id] = singleEntry
if (singleEntry.nameHash != -1) {
val nameHashTable = Int2ObjectOpenHashMap<IntSortedSet>()
nameHashTable[singleEntry.nameHash] = IntSortedSets.singleton(singleEntry.id)
this.nameHashTable = nameHashTable
}
}
this.singleEntry = null
this.entries = entries
}
entries[id] = entry
return entry
}
public fun createOrGetNamed(nameHash: Int): T {
var entry = getNamed(nameHash)
if (entry != null) {
return entry
}
entry = createOrGet(allocateId())
entry.nameHash = nameHash
return entry
}
public fun createOrGet(name: String): T {
return createOrGetNamed(name.krHashCode())
}
public fun remove(id: Int): T? {
val entry = get(id)
entry?.remove()
return entry
}
public fun removeNamed(nameHash: Int): T? {
val entry = getNamed(nameHash)
entry?.remove()
return entry
}
public fun remove(name: String): T? {
return removeNamed(name.krHashCode())
}
private fun allocateId(): Int {
val size = size
val capacity = capacity
if (size == capacity) {
return capacity
}
val singleEntry = singleEntry
if (singleEntry != null) {
assert(singleEntry.id != 0)
return 0
}
val entries = entries!!
for (id in 0 until capacity) {
if (!entries.containsKey(id)) {
return id
}
}
throw AssertionError()
}
internal fun rename(id: Int, prevNameHash: Int, newNameHash: Int) {
if (prevNameHash == newNameHash || singleEntry != null) {
return
}
var nameHashTable = nameHashTable
if (nameHashTable != null && prevNameHash != -1) {
val set = nameHashTable.get(prevNameHash)
assert(set != null && set.contains(id))
if (set.size > 1) {
set.remove(id)
} else {
nameHashTable.remove(prevNameHash)
}
}
if (newNameHash != -1) {
if (nameHashTable == null) {
nameHashTable = Int2ObjectOpenHashMap()
}
val set = nameHashTable[newNameHash]
when (set) {
null -> nameHashTable[newNameHash] = IntSortedSets.singleton(id)
is IntSortedSets.Singleton -> {
val newSet = IntAVLTreeSet()
newSet.add(set.firstInt())
newSet.add(id)
nameHashTable[newNameHash] = newSet
}
else -> set.add(id)
}
this.nameHashTable = nameHashTable
}
}
internal fun remove(entry: T) {
if (singleEntry?.id == entry.id) {
singleEntry = null
return
}
val entries = entries
check(entries != null)
val removedEntry = entries.remove(entry.id)
check(removedEntry != null)
if (entries.size == 1) {
val firstId = entries.firstIntKey()
singleEntry = entries[firstId]
this.entries = null
nameHashTable = null
return
}
rename(entry.id, entry.nameHash, -1)
}
override fun iterator(): MutableIterator<T> {
val singleEntry = singleEntry
if (singleEntry != null) {
return object : MutableIterator<T> {
private var pos = 0
private var removed = false
override fun hasNext(): Boolean {
return pos == 0
}
override fun next(): T {
if (pos++ != 0) {
throw NoSuchElementException()
}
return singleEntry
}
override fun remove() {
check(pos == 1 && !removed)
removed = true
singleEntry.remove()
}
}
}
val entries = entries
if (entries != null) {
return object : MutableIterator<T> {
private val it = entries.values.iterator()
private var last: T? = null
override fun hasNext(): Boolean {
return it.hasNext()
}
override fun next(): T {
last = null
val entry = it.next()
last = entry
return entry
}
override fun remove() {
val last = last
check(last != null)
last.remove()
this.last = null
}
}
}
return object : MutableIterator<T> {
override fun hasNext(): Boolean {
return false
}
override fun next(): T {
throw NoSuchElementException()
}
override fun remove() {
throw IllegalStateException()
}
}
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as MutableNamedEntryCollection<*>
if (singleEntry != other.singleEntry) return false
if (entries != other.entries) return false
return true
}
override fun hashCode(): Int {
var result = singleEntry?.hashCode() ?: 0
result = 31 * result + (entries?.hashCode() ?: 0)
return result
}
}

@ -1,18 +1,6 @@
package org.openrs2.cache package org.openrs2.cache
import org.openrs2.util.krHashCode
public interface NamedEntry { public interface NamedEntry {
public val id: Int public val id: Int
public var nameHash: Int public val nameHash: Int
public fun setName(name: String) {
nameHash = name.krHashCode()
}
public fun clearName() {
nameHash = -1
}
public fun remove()
} }

@ -1,379 +1,25 @@
package org.openrs2.cache package org.openrs2.cache
import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap
import it.unimi.dsi.fastutil.ints.Int2ObjectMap
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap
import it.unimi.dsi.fastutil.ints.IntAVLTreeSet
import it.unimi.dsi.fastutil.ints.IntSortedSet
import it.unimi.dsi.fastutil.ints.IntSortedSets
import org.openrs2.util.krHashCode import org.openrs2.util.krHashCode
/** /**
* A specialist collection type entirely designed for use by the [Js5Index] * A read-only view of a [MutableNamedEntryCollection].
* class.
*
* Entries may be accessed by their ID number or by the hash of their name, if
* they are named.
*
* Entries are sorted by their ID. The IDs should be fairly dense starting
* around 0, as the client stores entries in an array. Nevertheless, this
* implementation currently supports sparse IDs efficiently as the entries are
* actually stored in a tree map, primarily to make manipulation of the
* collection simpler (the client's implementation has the luxury of being
* read-only!)
*
* As the vast majority of [Js5Index.Group]s only have a single [Js5Index.File]
* entry, there is a special case for single entry collections. This avoids the
* memory overhead of the underlying collections in this special case, at the
* expense of making the logic slightly more complicated.
*
* Entries without a name have a name hash of -1. This is consistent with the
* client, which sets the name hash array to -1 before populating it. If the
* index or group is sparse, any non-existent IDs still have a hash of -1, which
* is passed into the client's IntHashTable class. This makes -1 unusable, so I
* feel this choice is justified.
*
* In practice, I think indexes generally either have names for every file or
* none at all, but allowing a mixture in this implementation makes mutating
* the index simpler.
*
* This implementation permits multiple entries to have the same name hash. In
* the event of a collision, the colliding entry with the lowest ID is returned
* from methods that look up an entry by its name hash. This implementation
* still behaves correctly if one of the colliding entries is removed: it will
* always return the entry with the next lowest ID.
*
* This is an edge case that is unlikely to be hit in practice: the 550 cache
* has no collisions, and the 876 cache only has a single collision (the hash
* for the empty string).
*/ */
public abstract class NamedEntryCollection<T : NamedEntry>( public interface NamedEntryCollection<out T : NamedEntry> : Iterable<T> {
private val entryConstructor: (NamedEntryCollection<T>, Int) -> T
) : MutableIterable<T> {
private var singleEntry: T? = null
private var entries: Int2ObjectAVLTreeMap<T>? = null
// XXX(gpe): unfortunately fastutil doesn't have a multimap type
private var nameHashTable: Int2ObjectMap<IntSortedSet>? = null
public val size: Int public val size: Int
get() {
if (singleEntry != null) {
return 1
}
val entries = entries ?: return 0
return entries.size
}
public val capacity: Int public val capacity: Int
get() {
val entry = singleEntry
if (entry != null) {
return entry.id + 1
}
val entries = entries ?: return 0
assert(entries.isNotEmpty())
return entries.lastIntKey() + 1
}
public operator fun contains(id: Int): Boolean {
require(id >= 0)
val entry = singleEntry
if (entry?.id == id) {
return true
}
val entries = entries ?: return false public operator fun contains(id: Int): Boolean
return entries.containsKey(id) public fun containsNamed(nameHash: Int): Boolean
}
public fun containsNamed(nameHash: Int): Boolean {
require(nameHash != -1)
val entry = singleEntry
if (entry?.nameHash == nameHash) {
return true
}
return nameHashTable?.containsKey(nameHash) ?: false
}
public operator fun contains(name: String): Boolean { public operator fun contains(name: String): Boolean {
return containsNamed(name.krHashCode()) return containsNamed(name.krHashCode())
} }
public operator fun get(id: Int): T? { public operator fun get(id: Int): T?
require(id >= 0) public fun getNamed(nameHash: Int): T?
val entry = singleEntry
if (entry?.id == id) {
return entry
}
val entries = entries ?: return null
return entries[id]
}
public fun getNamed(nameHash: Int): T? {
require(nameHash != -1)
val entry = singleEntry
if (entry?.nameHash == nameHash) {
return entry
}
val nameHashTable = nameHashTable ?: return null
val ids = nameHashTable.getOrDefault(nameHash, IntSortedSets.EMPTY_SET)
return if (ids.isNotEmpty()) {
get(ids.firstInt())!!
} else {
null
}
}
public operator fun get(name: String): T? { public operator fun get(name: String): T? {
return getNamed(name.krHashCode()) return getNamed(name.krHashCode())
} }
public fun createOrGet(id: Int): T {
var entry = get(id)
if (entry != null) {
return entry
}
entry = entryConstructor(this, id)
val singleEntry = singleEntry
var entries = entries
if (singleEntry == null && entries == null) {
this.singleEntry = entry
return entry
}
if (entries == null) {
entries = Int2ObjectAVLTreeMap<T>()
if (singleEntry != null) {
entries[singleEntry.id] = singleEntry
if (singleEntry.nameHash != -1) {
val nameHashTable = Int2ObjectOpenHashMap<IntSortedSet>()
nameHashTable[singleEntry.nameHash] = IntSortedSets.singleton(singleEntry.id)
this.nameHashTable = nameHashTable
}
}
this.singleEntry = null
this.entries = entries
}
entries[id] = entry
return entry
}
public fun createOrGetNamed(nameHash: Int): T {
var entry = getNamed(nameHash)
if (entry != null) {
return entry
}
entry = createOrGet(allocateId())
entry.nameHash = nameHash
return entry
}
public fun createOrGet(name: String): T {
return createOrGetNamed(name.krHashCode())
}
public fun remove(id: Int): T? {
val entry = get(id)
entry?.remove()
return entry
}
public fun removeNamed(nameHash: Int): T? {
val entry = getNamed(nameHash)
entry?.remove()
return entry
}
public fun remove(name: String): T? {
return removeNamed(name.krHashCode())
}
private fun allocateId(): Int {
val size = size
val capacity = capacity
if (size == capacity) {
return capacity
}
val singleEntry = singleEntry
if (singleEntry != null) {
assert(singleEntry.id != 0)
return 0
}
val entries = entries!!
for (id in 0 until capacity) {
if (!entries.containsKey(id)) {
return id
}
}
throw AssertionError()
}
internal fun rename(id: Int, prevNameHash: Int, newNameHash: Int) {
if (prevNameHash == newNameHash || singleEntry != null) {
return
}
var nameHashTable = nameHashTable
if (nameHashTable != null && prevNameHash != -1) {
val set = nameHashTable.get(prevNameHash)
assert(set != null && set.contains(id))
if (set.size > 1) {
set.remove(id)
} else {
nameHashTable.remove(prevNameHash)
}
}
if (newNameHash != -1) {
if (nameHashTable == null) {
nameHashTable = Int2ObjectOpenHashMap()
}
val set = nameHashTable[newNameHash]
when (set) {
null -> nameHashTable[newNameHash] = IntSortedSets.singleton(id)
is IntSortedSets.Singleton -> {
val newSet = IntAVLTreeSet()
newSet.add(set.firstInt())
newSet.add(id)
nameHashTable[newNameHash] = newSet
}
else -> set.add(id)
}
this.nameHashTable = nameHashTable
}
}
internal fun remove(entry: T) {
if (singleEntry?.id == entry.id) {
singleEntry = null
return
}
val entries = entries
check(entries != null)
val removedEntry = entries.remove(entry.id)
check(removedEntry != null)
if (entries.size == 1) {
val firstId = entries.firstIntKey()
singleEntry = entries[firstId]
this.entries = null
nameHashTable = null
return
}
rename(entry.id, entry.nameHash, -1)
}
override fun iterator(): MutableIterator<T> {
val singleEntry = singleEntry
if (singleEntry != null) {
return object : MutableIterator<T> {
private var pos = 0
private var removed = false
override fun hasNext(): Boolean {
return pos == 0
}
override fun next(): T {
if (pos++ != 0) {
throw NoSuchElementException()
}
return singleEntry
}
override fun remove() {
check(pos == 1 && !removed)
removed = true
singleEntry.remove()
}
}
}
val entries = entries
if (entries != null) {
return object : MutableIterator<T> {
private val it = entries.values.iterator()
private var last: T? = null
override fun hasNext(): Boolean {
return it.hasNext()
}
override fun next(): T {
last = null
val entry = it.next()
last = entry
return entry
}
override fun remove() {
val last = last
check(last != null)
last.remove()
this.last = null
}
}
}
return object : MutableIterator<T> {
override fun hasNext(): Boolean {
return false
}
override fun next(): T {
throw NoSuchElementException()
}
override fun remove() {
throw IllegalStateException()
}
}
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as NamedEntryCollection<*>
if (singleEntry != other.singleEntry) return false
if (entries != other.entries) return false
return true
}
override fun hashCode(): Int {
var result = singleEntry?.hashCode() ?: 0
result = 31 * result + (entries?.hashCode() ?: 0)
return result
}
} }

@ -10,10 +10,10 @@ import kotlin.test.assertTrue
class NamedEntryCollectionTest { class NamedEntryCollectionTest {
private class TestEntry( private class TestEntry(
parent: NamedEntryCollection<TestEntry>, parent: MutableNamedEntryCollection<TestEntry>,
override val id: Int override val id: Int
) : NamedEntry { ) : MutableNamedEntry {
private var parent: NamedEntryCollection<TestEntry>? = parent private var parent: MutableNamedEntryCollection<TestEntry>? = parent
override var nameHash: Int = -1 override var nameHash: Int = -1
set(value) { set(value) {
@ -27,7 +27,7 @@ class NamedEntryCollectionTest {
} }
} }
private class TestCollection : NamedEntryCollection<TestEntry>(::TestEntry) private class TestCollection : MutableNamedEntryCollection<TestEntry>(::TestEntry)
@Test @Test
fun testBounds() { fun testBounds() {

Loading…
Cancel
Save