gary.tierney
/
openrs2
forked from openrs2/openrs2
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for the current master index format

Browse Source 
I'm not sure if the auto-detection code works: I'm assuming that the new
format was introduced at the same time as the lengths flag in Js5Index,
but I haven't confirmed this.

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent 192b24b9bd
commit 53db5b3314
5 changed files with 118 additions and 30 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      archive/src/main/resources/org/openrs2/archive/V1__init.sql
  2. 46
      cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt
  3. 3
      cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt
  4. 96
      cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt
  5. BIN
      cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat

3
archive/src/main/resources/org/openrs2/archive/V1__init.sql
Unescape View File

@ -93,7 +93,8 @@ CREATE TABLE index_files (
CREATE TYPE master_index_format AS ENUM (
'original',
'versioned',
'whirlpool'
'whirlpool',
'lengths'
);
CREATE TABLE master_indexes (

46
cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt vendored
Unescape View File

@ -17,6 +17,8 @@ public data class Js5MasterIndex(
public class Entry(
public var version: Int,
public var checksum: Int,
public var groups: Int,
public var totalUncompressedLength: Int,
digest: ByteArray?
) {
public var digest: ByteArray? = digest
@ -33,6 +35,8 @@ public data class Js5MasterIndex(
if (version != other.version) return false
if (checksum != other.checksum) return false
if (groups != other.groups) return false
if (totalUncompressedLength != other.totalUncompressedLength) return false
if (digest != null) {
if (other.digest == null) return false
if (!digest.contentEquals(other.digest)) return false
@ -44,6 +48,8 @@ public data class Js5MasterIndex(
override fun hashCode(): Int {
var result = version
result = 31 * result + checksum
result = 31 * result + groups
result = 31 * result + totalUncompressedLength
result = 31 * result + (digest?.contentHashCode() ?: 0)
return result
}
@ -55,7 +61,8 @@ public data class Js5MasterIndex(
} else {
"null"
}
return "Entry(version=$version, checksum=$checksum, digest=$hex)"
return "Entry(version=$version, checksum=$checksum, groups=$groups, " +
"totalUncompressedLength=$totalUncompressedLength, digest=$hex)"
}
}
@ -73,6 +80,11 @@ public data class Js5MasterIndex(
buf.writeInt(entry.version)
}
if (format >= MasterIndexFormat.LENGTHS) {
buf.writeInt(entry.groups)
buf.writeInt(entry.totalUncompressedLength)
}
if (format >= MasterIndexFormat.WHIRLPOOL) {
val digest = entry.digest
if (digest != null) {
@ -116,27 +128,31 @@ public data class Js5MasterIndex(
* entries with a zero CRC are probably invalid.
*/
for (i in nextArchive until archive) {
masterIndex.entries += Entry(0, 0, null)
masterIndex.entries += Entry(0, 0, 0, 0, null)
}
val entry = store.read(Js5Archive.ARCHIVESET, archive).use { buf ->
val checksum = buf.crc32()
val digest = buf.whirlpool()
val version = Js5Compression.uncompress(buf).use { uncompressed ->
Js5Compression.uncompress(buf).use { uncompressed ->
val index = Js5Index.read(uncompressed)
if (index.hasDigests) {
if (index.hasLengths) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.LENGTHS)
} else if (index.hasDigests) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.WHIRLPOOL)
} else if (index.protocol >= Js5Protocol.VERSIONED) {
masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.VERSIONED)
}
index.version
}
val version = index.version
val groups = index.size
val totalUncompressedLength = index.sumBy(Js5Index.Group::uncompressedLength)
// TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above?
Entry(version, checksum, digest)
// TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above?
Entry(version, checksum, groups, totalUncompressedLength, digest)
}
}
masterIndex.entries += entry
@ -165,7 +181,7 @@ public data class Js5MasterIndex(
}
len / 8
}
MasterIndexFormat.WHIRLPOOL -> {
else -> {
buf.readUnsignedByte().toInt()
}
}
@ -179,6 +195,16 @@ public data class Js5MasterIndex(
0
}
val groups: Int
val totalUncompressedLength: Int
if (format >= MasterIndexFormat.LENGTHS) {
groups = buf.readInt()
totalUncompressedLength = buf.readInt()
} else {
groups = 0
totalUncompressedLength = 0
}
val digest = if (format >= MasterIndexFormat.WHIRLPOOL) {
val bytes = ByteArray(Whirlpool.DIGESTBYTES)
buf.readBytes(bytes)
@ -187,7 +213,7 @@ public data class Js5MasterIndex(
null
}
index.entries += Entry(version, checksum, digest)
index.entries += Entry(version, checksum, groups, totalUncompressedLength, digest)
}
val end = buf.readerIndex()

3
cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt vendored
Unescape View File

@ -3,5 +3,6 @@ package org.openrs2.cache
public enum class MasterIndexFormat {
ORIGINAL,
VERSIONED,
WHIRLPOOL
WHIRLPOOL,
LENGTHS
}

96
cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt vendored
Unescape View File

@ -5,6 +5,7 @@ import io.netty.buffer.ByteBufUtil
import io.netty.buffer.Unpooled
import org.openrs2.buffer.use
import org.openrs2.crypto.Rsa
import org.openrs2.crypto.Whirlpool
import java.nio.file.Path
import kotlin.test.Test
import kotlin.test.assertEquals
@ -22,7 +23,7 @@ class Js5MasterIndexTest {
MasterIndexFormat.ORIGINAL,
mutableListOf(
Js5MasterIndex.Entry(
0, 609698396, ByteBufUtil.decodeHexDump(
0, 609698396, 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
)
@ -43,28 +44,28 @@ class Js5MasterIndexTest {
MasterIndexFormat.VERSIONED,
mutableListOf(
Js5MasterIndex.Entry(
0, 609698396, ByteBufUtil.decodeHexDump(
0, 609698396, 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
)
),
Js5MasterIndex.Entry(
0x12345678, 78747481, ByteBufUtil.decodeHexDump(
0x12345678, 78747481, 0, 0, ByteBufUtil.decodeHexDump(
"180ff4ad371f56d4a90d81e0b69b23836cd9b101b828f18b7e6d232c4d302539" +
"638eb2e9259957645aae294f09b2d669c93dbbfc0d8359f1b232ae468f678ca1"
)
),
Js5MasterIndex.Entry(0, 0, null),
Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry(
0x9ABCDEF0.toInt(), -456081154, ByteBufUtil.decodeHexDump(
0x9ABCDEF0.toInt(), -456081154, 0, 0, ByteBufUtil.decodeHexDump(
"972003261b7628525346e0052567662e5695147ad710f877b63b9ab53b3f6650" +
"ca003035fde4398b2ef73a60e4b13798aa597a30c1bf0a13c0cd412394af5f96"
)
),
Js5MasterIndex.Entry(0, 0, null),
Js5MasterIndex.Entry(0, 0, null),
Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry(0, 0, 0, 0, null),
Js5MasterIndex.Entry(
0xAA55AA55.toInt(), 186613982, ByteBufUtil.decodeHexDump(
0xAA55AA55.toInt(), 186613982, 0, 0, ByteBufUtil.decodeHexDump(
"d50a6e9abd3b5269606304dc2769cbc8618e1ae6ff705291c0dfcc374e450dd2" +
"5f1be5f1d5459651d22d3e87ef0a1c69be7807f661cd001be24a6609f6d57916"
)
@ -85,13 +86,13 @@ class Js5MasterIndexTest {
MasterIndexFormat.WHIRLPOOL,
mutableListOf(
Js5MasterIndex.Entry(
0, 668177970, ByteBufUtil.decodeHexDump(
0, 668177970, 0, 0, ByteBufUtil.decodeHexDump(
"2faa83116e1d1719d5db15f128eb57f62afbf0207c47bced3f558ec17645d138" +
"72f4fb9b0e36a5f6f5d30e1295b3fa49556dfd0819cb5137f3b69f64155f3fb7"
)
),
Js5MasterIndex.Entry(
0, 1925442845, ByteBufUtil.decodeHexDump(
0, 1925442845, 0, 0, ByteBufUtil.decodeHexDump(
"fcc45b0ab6d0067889e44de0004bcbb6cc538aff8f80edf1b49b583cedd73fea" +
"937ae6990235257fe8aa35c44d35450c13e670711337ee5116957cd98cc27985"
)
@ -101,6 +102,27 @@ class Js5MasterIndexTest {
)
}
@Test
fun testCreateLengths() {
val index = Store.open(ROOT.resolve("lengths")).use { store ->
Js5MasterIndex.create(store)
}
assertEquals(
Js5MasterIndex(
MasterIndexFormat.LENGTHS,
mutableListOf(
Js5MasterIndex.Entry(
0x12345678, -1080883457, 3, 123, ByteBufUtil.decodeHexDump(
"0bf30b80b7213154ada5c3797be15a8fbb6a96a80432e2093e10617bcb4e67de" +
"9a858211cabe844c6fa3a1fbfe3164a3e4e1918983c69597dff3fc3c53096884"
)
)
)
), index
)
}
@Test
fun testReadOriginal() {
Unpooled.wrappedBuffer(encodedOriginal).use { buf ->
@ -242,6 +264,25 @@ class Js5MasterIndexTest {
}
}
@Test
fun testReadLengths() {
Unpooled.wrappedBuffer(encodedLengths).use { buf ->
val index = Js5MasterIndex.read(buf, MasterIndexFormat.LENGTHS)
assertEquals(decodedLengths, index)
}
}
@Test
fun testWriteLengths() {
ByteBufAllocator.DEFAULT.buffer().use { actual ->
decodedLengths.write(actual)
Unpooled.wrappedBuffer(encodedLengths).use { expected ->
assertEquals(expected, actual)
}
}
}
private companion object {
private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI())
private val PRIVATE_KEY = Rsa.readPrivateKey(ROOT.resolve("private.key"))
@ -251,9 +292,9 @@ class Js5MasterIndexTest {
private val decodedOriginal = Js5MasterIndex(
MasterIndexFormat.ORIGINAL,
mutableListOf(
Js5MasterIndex.Entry(0, 1, null),
Js5MasterIndex.Entry(0, 3, null),
Js5MasterIndex.Entry(0, 5, null)
Js5MasterIndex.Entry(0, 1, 0, 0, null),
Js5MasterIndex.Entry(0, 3, 0, 0, null),
Js5MasterIndex.Entry(0, 5, 0, 0, null)
)
)
@ -261,9 +302,9 @@ class Js5MasterIndexTest {
private val decodedVersioned = Js5MasterIndex(
MasterIndexFormat.VERSIONED,
mutableListOf(
Js5MasterIndex.Entry(0, 1, null),
Js5MasterIndex.Entry(2, 3, null),
Js5MasterIndex.Entry(4, 5, null)
Js5MasterIndex.Entry(0, 1, 0, 0, null),
Js5MasterIndex.Entry(2, 3, 0, 0, null),
Js5MasterIndex.Entry(4, 5, 0, 0, null)
)
)
@ -281,7 +322,7 @@ class Js5MasterIndexTest {
MasterIndexFormat.WHIRLPOOL,
mutableListOf(
Js5MasterIndex.Entry(
0x01234567, 0x89ABCDEF.toInt(), ByteBufUtil.decodeHexDump(
0x01234567, 0x89ABCDEF.toInt(), 0, 0, ByteBufUtil.decodeHexDump(
"0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" +
"4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c"
)
@ -302,7 +343,7 @@ class Js5MasterIndexTest {
private val decodedWhirlpoolNullDigest = Js5MasterIndex(
MasterIndexFormat.WHIRLPOOL,
mutableListOf(
Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), null)
Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), 0, 0, null)
)
)
@ -321,5 +362,24 @@ class Js5MasterIndexTest {
"57707ce3f4f5a7af8471eda5c0c0748454a9cbb48c25ebe4e7fd94e3881b6461" +
"d06e2bce128dc96decb537b8e9611591d445d7dfd3701d25ac05f8d091581aef"
)
private val decodedLengths = Js5MasterIndex(
MasterIndexFormat.LENGTHS,
mutableListOf(
Js5MasterIndex.Entry(0x012345678, 0x89ABCDEF.toInt(), 3, 123, ByteArray(Whirlpool.DIGESTBYTES))
)
)
private val encodedLengths = ByteBufUtil.decodeHexDump(
"01" +
"89abcdef" +
"12345678" +
"00000003" +
"0000007b" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0a" +
"3d9f704a2b2e1b6f4e7b3a9b558baed7ccb10787a754b6fd36acb77ba3491726" +
"fef29e470218a98693bfc1b98a611f15e0b35a11bd181830ff4912377653a87a"
)
}
}

BIN
cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat vendored
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save