Add support for the current master index format

I'm not sure if the auto-detection code works: I'm assuming that the new
format was introduced at the same time as the lengths flag in Js5Index,
but I haven't confirmed this.

Signed-off-by: Graham <gpe@openrs2.org>
bzip2
Graham 4 years ago
parent 192b24b9bd
commit 53db5b3314
  1. 3
      archive/src/main/resources/org/openrs2/archive/V1__init.sql
  2. 46
      cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt
  3. 3
      cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt
  4. 96
      cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt
  5. BIN
      cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat

@ -93,7 +93,8 @@ CREATE TABLE index_files (
CREATE TYPE master_index_format AS ENUM ( CREATE TYPE master_index_format AS ENUM (
'original', 'original',
'versioned', 'versioned',
'whirlpool' 'whirlpool',
'lengths'
); );
CREATE TABLE master_indexes ( CREATE TABLE master_indexes (

@ -17,6 +17,8 @@ public data class Js5MasterIndex(
public class Entry( public class Entry(
public var version: Int, public var version: Int,
public var checksum: Int, public var checksum: Int,
public var groups: Int,
public var totalUncompressedLength: Int,
digest: ByteArray? digest: ByteArray?
) { ) {
public var digest: ByteArray? = digest public var digest: ByteArray? = digest
@ -33,6 +35,8 @@ public data class Js5MasterIndex(
if (version != other.version) return false if (version != other.version) return false
if (checksum != other.checksum) return false if (checksum != other.checksum) return false
if (groups != other.groups) return false
if (totalUncompressedLength != other.totalUncompressedLength) return false
if (digest != null) { if (digest != null) {
if (other.digest == null) return false if (other.digest == null) return false
if (!digest.contentEquals(other.digest)) return false if (!digest.contentEquals(other.digest)) return false
@ -44,6 +48,8 @@ public data class Js5MasterIndex(
override fun hashCode(): Int { override fun hashCode(): Int {
var result = version var result = version
result = 31 * result + checksum result = 31 * result + checksum
result = 31 * result + groups
result = 31 * result + totalUncompressedLength
result = 31 * result + (digest?.contentHashCode() ?: 0) result = 31 * result + (digest?.contentHashCode() ?: 0)
return result return result
} }
@ -55,7 +61,8 @@ public data class Js5MasterIndex(
} else { } else {
"null" "null"
} }
return "Entry(version=$version, checksum=$checksum, digest=$hex)" return "Entry(version=$version, checksum=$checksum, groups=$groups, " +
"totalUncompressedLength=$totalUncompressedLength, digest=$hex)"
} }
} }
@ -73,6 +80,11 @@ public data class Js5MasterIndex(
buf.writeInt(entry.version) buf.writeInt(entry.version)
} }
if (format >= MasterIndexFormat.LENGTHS) {
buf.writeInt(entry.groups)
buf.writeInt(entry.totalUncompressedLength)
}
if (format >= MasterIndexFormat.WHIRLPOOL) { if (format >= MasterIndexFormat.WHIRLPOOL) {
val digest = entry.digest val digest = entry.digest
if (digest != null) { if (digest != null) {
@ -116,27 +128,31 @@ public data class Js5MasterIndex(
* entries with a zero CRC are probably invalid. * entries with a zero CRC are probably invalid.
*/ */
for (i in nextArchive until archive) { for (i in nextArchive until archive) {
masterIndex.entries += Entry(0, 0, null) masterIndex.entries += Entry(0, 0, 0, 0, null)
} }
val entry = store.read(Js5Archive.ARCHIVESET, archive).use { buf -> val entry = store.read(Js5Archive.ARCHIVESET, archive).use { buf ->
val checksum = buf.crc32() val checksum = buf.crc32()
val digest = buf.whirlpool() val digest = buf.whirlpool()
val version = Js5Compression.uncompress(buf).use { uncompressed -> Js5Compression.uncompress(buf).use { uncompressed ->
val index = Js5Index.read(uncompressed) val index = Js5Index.read(uncompressed)
if (index.hasDigests) { if (index.hasLengths) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.LENGTHS)
} else if (index.hasDigests) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.WHIRLPOOL) masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.WHIRLPOOL)
} else if (index.protocol >= Js5Protocol.VERSIONED) { } else if (index.protocol >= Js5Protocol.VERSIONED) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.VERSIONED) masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.VERSIONED)
} }
index.version val version = index.version
} val groups = index.size
val totalUncompressedLength = index.sumBy(Js5Index.Group::uncompressedLength)
// TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above? // TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above?
Entry(version, checksum, digest) Entry(version, checksum, groups, totalUncompressedLength, digest)
}
} }
masterIndex.entries += entry masterIndex.entries += entry
@ -165,7 +181,7 @@ public data class Js5MasterIndex(
} }
len / 8 len / 8
} }
MasterIndexFormat.WHIRLPOOL -> { else -> {
buf.readUnsignedByte().toInt() buf.readUnsignedByte().toInt()
} }
} }
@ -179,6 +195,16 @@ public data class Js5MasterIndex(
0 0
} }
val groups: Int
val totalUncompressedLength: Int
if (format >= MasterIndexFormat.LENGTHS) {
groups = buf.readInt()
totalUncompressedLength = buf.readInt()
} else {
groups = 0
totalUncompressedLength = 0
}
val digest = if (format >= MasterIndexFormat.WHIRLPOOL) { val digest = if (format >= MasterIndexFormat.WHIRLPOOL) {
val bytes = ByteArray(Whirlpool.DIGESTBYTES) val bytes = ByteArray(Whirlpool.DIGESTBYTES)
buf.readBytes(bytes) buf.readBytes(bytes)
@ -187,7 +213,7 @@ public data class Js5MasterIndex(
null null
} }
index.entries += Entry(version, checksum, digest) index.entries += Entry(version, checksum, groups, totalUncompressedLength, digest)
} }
val end = buf.readerIndex() val end = buf.readerIndex()

@ -3,5 +3,6 @@ package org.openrs2.cache
public enum class MasterIndexFormat { public enum class MasterIndexFormat {
ORIGINAL, ORIGINAL,
VERSIONED, VERSIONED,
WHIRLPOOL WHIRLPOOL,
LENGTHS
} }

@ -5,6 +5,7 @@ import io.netty.buffer.ByteBufUtil
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import org.openrs2.buffer.use import org.openrs2.buffer.use
import org.openrs2.crypto.Rsa import org.openrs2.crypto.Rsa
import org.openrs2.crypto.Whirlpool
import java.nio.file.Path import java.nio.file.Path
import kotlin.test.Test import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
@ -22,7 +23,7 @@ class Js5MasterIndexTest {
MasterIndexFormat.ORIGINAL, MasterIndexFormat.ORIGINAL,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0, 609698396, ByteBufUtil.decodeHexDump( 0, 609698396, 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
) )
@ -43,28 +44,28 @@ class Js5MasterIndexTest {
MasterIndexFormat.VERSIONED, MasterIndexFormat.VERSIONED,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0, 609698396, ByteBufUtil.decodeHexDump( 0, 609698396, 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
) )
), ),
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0x12345678, 78747481, ByteBufUtil.decodeHexDump( 0x12345678, 78747481, 0, 0, ByteBufUtil.decodeHexDump(
"180ff4ad371f56d4a90d81e0b69b23836cd9b101b828f18b7e6d232c4d302539" + "180ff4ad371f56d4a90d81e0b69b23836cd9b101b828f18b7e6d232c4d302539" +
"638eb2e9259957645aae294f09b2d669c93dbbfc0d8359f1b232ae468f678ca1" "638eb2e9259957645aae294f09b2d669c93dbbfc0d8359f1b232ae468f678ca1"
) )
), ),
Js5MasterIndex.Entry(0, 0, null), Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0x9ABCDEF0.toInt(), -456081154, ByteBufUtil.decodeHexDump( 0x9ABCDEF0.toInt(), -456081154, 0, 0, ByteBufUtil.decodeHexDump(
"972003261b7628525346e0052567662e5695147ad710f877b63b9ab53b3f6650" + "972003261b7628525346e0052567662e5695147ad710f877b63b9ab53b3f6650" +
"ca003035fde4398b2ef73a60e4b13798aa597a30c1bf0a13c0cd412394af5f96" "ca003035fde4398b2ef73a60e4b13798aa597a30c1bf0a13c0cd412394af5f96"
) )
), ),
Js5MasterIndex.Entry(0, 0, null), Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry(0, 0, null), Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0xAA55AA55.toInt(), 186613982, ByteBufUtil.decodeHexDump( 0xAA55AA55.toInt(), 186613982, 0, 0, ByteBufUtil.decodeHexDump(
"d50a6e9abd3b5269606304dc2769cbc8618e1ae6ff705291c0dfcc374e450dd2" + "d50a6e9abd3b5269606304dc2769cbc8618e1ae6ff705291c0dfcc374e450dd2" +
"5f1be5f1d5459651d22d3e87ef0a1c69be7807f661cd001be24a6609f6d57916" "5f1be5f1d5459651d22d3e87ef0a1c69be7807f661cd001be24a6609f6d57916"
) )
@ -85,13 +86,13 @@ class Js5MasterIndexTest {
MasterIndexFormat.WHIRLPOOL, MasterIndexFormat.WHIRLPOOL,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0, 668177970, ByteBufUtil.decodeHexDump( 0, 668177970, 0, 0, ByteBufUtil.decodeHexDump(
"2faa83116e1d1719d5db15f128eb57f62afbf0207c47bced3f558ec17645d138" + "2faa83116e1d1719d5db15f128eb57f62afbf0207c47bced3f558ec17645d138" +
"72f4fb9b0e36a5f6f5d30e1295b3fa49556dfd0819cb5137f3b69f64155f3fb7" "72f4fb9b0e36a5f6f5d30e1295b3fa49556dfd0819cb5137f3b69f64155f3fb7"
) )
), ),
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0, 1925442845, ByteBufUtil.decodeHexDump( 0, 1925442845, 0, 0, ByteBufUtil.decodeHexDump(
"fcc45b0ab6d0067889e44de0004bcbb6cc538aff8f80edf1b49b583cedd73fea" + "fcc45b0ab6d0067889e44de0004bcbb6cc538aff8f80edf1b49b583cedd73fea" +
"937ae6990235257fe8aa35c44d35450c13e670711337ee5116957cd98cc27985" "937ae6990235257fe8aa35c44d35450c13e670711337ee5116957cd98cc27985"
) )
@ -101,6 +102,27 @@ class Js5MasterIndexTest {
) )
} }
@Test
fun testCreateLengths() {
val index = Store.open(ROOT.resolve("lengths")).use { store ->
Js5MasterIndex.create(store)
}
assertEquals(
Js5MasterIndex(
MasterIndexFormat.LENGTHS,
mutableListOf(
Js5MasterIndex.Entry(
0x12345678, -1080883457, 3, 123, ByteBufUtil.decodeHexDump(
"0bf30b80b7213154ada5c3797be15a8fbb6a96a80432e2093e10617bcb4e67de" +
"9a858211cabe844c6fa3a1fbfe3164a3e4e1918983c69597dff3fc3c53096884"
)
)
)
), index
)
}
@Test @Test
fun testReadOriginal() { fun testReadOriginal() {
Unpooled.wrappedBuffer(encodedOriginal).use { buf -> Unpooled.wrappedBuffer(encodedOriginal).use { buf ->
@ -242,6 +264,25 @@ class Js5MasterIndexTest {
} }
} }
@Test
fun testReadLengths() {
Unpooled.wrappedBuffer(encodedLengths).use { buf ->
val index = Js5MasterIndex.read(buf, MasterIndexFormat.LENGTHS)
assertEquals(decodedLengths, index)
}
}
@Test
fun testWriteLengths() {
ByteBufAllocator.DEFAULT.buffer().use { actual ->
decodedLengths.write(actual)
Unpooled.wrappedBuffer(encodedLengths).use { expected ->
assertEquals(expected, actual)
}
}
}
private companion object { private companion object {
private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI()) private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI())
private val PRIVATE_KEY = Rsa.readPrivateKey(ROOT.resolve("private.key")) private val PRIVATE_KEY = Rsa.readPrivateKey(ROOT.resolve("private.key"))
@ -251,9 +292,9 @@ class Js5MasterIndexTest {
private val decodedOriginal = Js5MasterIndex( private val decodedOriginal = Js5MasterIndex(
MasterIndexFormat.ORIGINAL, MasterIndexFormat.ORIGINAL,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry(0, 1, null), Js5MasterIndex.Entry(0, 1, 0, 0, null),
Js5MasterIndex.Entry(0, 3, null), Js5MasterIndex.Entry(0, 3, 0, 0, null),
Js5MasterIndex.Entry(0, 5, null) Js5MasterIndex.Entry(0, 5, 0, 0, null)
) )
) )
@ -261,9 +302,9 @@ class Js5MasterIndexTest {
private val decodedVersioned = Js5MasterIndex( private val decodedVersioned = Js5MasterIndex(
MasterIndexFormat.VERSIONED, MasterIndexFormat.VERSIONED,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry(0, 1, null), Js5MasterIndex.Entry(0, 1, 0, 0, null),
Js5MasterIndex.Entry(2, 3, null), Js5MasterIndex.Entry(2, 3, 0, 0, null),
Js5MasterIndex.Entry(4, 5, null) Js5MasterIndex.Entry(4, 5, 0, 0, null)
) )
) )
@ -281,7 +322,7 @@ class Js5MasterIndexTest {
MasterIndexFormat.WHIRLPOOL, MasterIndexFormat.WHIRLPOOL,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry( Js5MasterIndex.Entry(
0x01234567, 0x89ABCDEF.toInt(), ByteBufUtil.decodeHexDump( 0x01234567, 0x89ABCDEF.toInt(), 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
) )
@ -302,7 +343,7 @@ class Js5MasterIndexTest {
private val decodedWhirlpoolNullDigest = Js5MasterIndex( private val decodedWhirlpoolNullDigest = Js5MasterIndex(
MasterIndexFormat.WHIRLPOOL, MasterIndexFormat.WHIRLPOOL,
mutableListOf( mutableListOf(
Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), null) Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), 0, 0, null)
) )
) )
@ -321,5 +362,24 @@ class Js5MasterIndexTest {
"57707ce3f4f5a7af8471eda5c0c0748454a9cbb48c25ebe4e7fd94e3881b6461" + "57707ce3f4f5a7af8471eda5c0c0748454a9cbb48c25ebe4e7fd94e3881b6461" +
"d06e2bce128dc96decb537b8e9611591d445d7dfd3701d25ac05f8d091581aef" "d06e2bce128dc96decb537b8e9611591d445d7dfd3701d25ac05f8d091581aef"
) )
private val decodedLengths = Js5MasterIndex(
MasterIndexFormat.LENGTHS,
mutableListOf(
Js5MasterIndex.Entry(0x012345678, 0x89ABCDEF.toInt(), 3, 123, ByteArray(Whirlpool.DIGESTBYTES))
)
)
private val encodedLengths = ByteBufUtil.decodeHexDump(
"01" +
"89abcdef" +
"12345678" +
"00000003" +
"0000007b" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0a" +
"3d9f704a2b2e1b6f4e7b3a9b558baed7ccb10787a754b6fd36acb77ba3491726" +
"fef29e470218a98693bfc1b98a611f15e0b35a11bd181830ff4912377653a87a"
)
} }
} }

Loading…
Cancel
Save