From 53db5b331483e5680ea89ede6c6196fc4e61f05a Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 13 Feb 2021 16:04:49 +0000 Subject: [PATCH] Add support for the current master index format I'm not sure if the auto-detection code works: I'm assuming that the new format was introduced at the same time as the lengths flag in Js5Index, but I haven't confirmed this. Signed-off-by: Graham --- .../org/openrs2/archive/V1__init.sql | 3 +- .../org/openrs2/cache/Js5MasterIndex.kt | 46 +++++++-- .../org/openrs2/cache/MasterIndexFormat.kt | 3 +- .../org/openrs2/cache/Js5MasterIndexTest.kt | 96 ++++++++++++++---- .../cache/master-index/lengths/255/0.dat | Bin 0 -> 85 bytes 5 files changed, 118 insertions(+), 30 deletions(-) create mode 100644 cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat diff --git a/archive/src/main/resources/org/openrs2/archive/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/V1__init.sql index d6b1924e..c03d0812 100644 --- a/archive/src/main/resources/org/openrs2/archive/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/V1__init.sql @@ -93,7 +93,8 @@ CREATE TABLE index_files ( CREATE TYPE master_index_format AS ENUM ( 'original', 'versioned', - 'whirlpool' + 'whirlpool', + 'lengths' ); CREATE TABLE master_indexes ( diff --git a/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt b/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt index 66745f80..2f09d6e0 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt @@ -17,6 +17,8 @@ public data class Js5MasterIndex( public class Entry( public var version: Int, public var checksum: Int, + public var groups: Int, + public var totalUncompressedLength: Int, digest: ByteArray? ) { public var digest: ByteArray? = digest @@ -33,6 +35,8 @@ public data class Js5MasterIndex( if (version != other.version) return false if (checksum != other.checksum) return false + if (groups != other.groups) return false + if (totalUncompressedLength != other.totalUncompressedLength) return false if (digest != null) { if (other.digest == null) return false if (!digest.contentEquals(other.digest)) return false @@ -44,6 +48,8 @@ public data class Js5MasterIndex( override fun hashCode(): Int { var result = version result = 31 * result + checksum + result = 31 * result + groups + result = 31 * result + totalUncompressedLength result = 31 * result + (digest?.contentHashCode() ?: 0) return result } @@ -55,7 +61,8 @@ public data class Js5MasterIndex( } else { "null" } - return "Entry(version=$version, checksum=$checksum, digest=$hex)" + return "Entry(version=$version, checksum=$checksum, groups=$groups, " + + "totalUncompressedLength=$totalUncompressedLength, digest=$hex)" } } @@ -73,6 +80,11 @@ public data class Js5MasterIndex( buf.writeInt(entry.version) } + if (format >= MasterIndexFormat.LENGTHS) { + buf.writeInt(entry.groups) + buf.writeInt(entry.totalUncompressedLength) + } + if (format >= MasterIndexFormat.WHIRLPOOL) { val digest = entry.digest if (digest != null) { @@ -116,27 +128,31 @@ public data class Js5MasterIndex( * entries with a zero CRC are probably invalid. */ for (i in nextArchive until archive) { - masterIndex.entries += Entry(0, 0, null) + masterIndex.entries += Entry(0, 0, 0, 0, null) } val entry = store.read(Js5Archive.ARCHIVESET, archive).use { buf -> val checksum = buf.crc32() val digest = buf.whirlpool() - val version = Js5Compression.uncompress(buf).use { uncompressed -> + Js5Compression.uncompress(buf).use { uncompressed -> val index = Js5Index.read(uncompressed) - if (index.hasDigests) { + if (index.hasLengths) { + masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.LENGTHS) + } else if (index.hasDigests) { masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.WHIRLPOOL) } else if (index.protocol >= Js5Protocol.VERSIONED) { masterIndex.format = maxOf(masterIndex.format, MasterIndexFormat.VERSIONED) } - index.version - } + val version = index.version + val groups = index.size + val totalUncompressedLength = index.sumBy(Js5Index.Group::uncompressedLength) - // TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above? - Entry(version, checksum, digest) + // TODO(gpe): should we throw an exception if there are trailing bytes here or in the block above? + Entry(version, checksum, groups, totalUncompressedLength, digest) + } } masterIndex.entries += entry @@ -165,7 +181,7 @@ public data class Js5MasterIndex( } len / 8 } - MasterIndexFormat.WHIRLPOOL -> { + else -> { buf.readUnsignedByte().toInt() } } @@ -179,6 +195,16 @@ public data class Js5MasterIndex( 0 } + val groups: Int + val totalUncompressedLength: Int + if (format >= MasterIndexFormat.LENGTHS) { + groups = buf.readInt() + totalUncompressedLength = buf.readInt() + } else { + groups = 0 + totalUncompressedLength = 0 + } + val digest = if (format >= MasterIndexFormat.WHIRLPOOL) { val bytes = ByteArray(Whirlpool.DIGESTBYTES) buf.readBytes(bytes) @@ -187,7 +213,7 @@ public data class Js5MasterIndex( null } - index.entries += Entry(version, checksum, digest) + index.entries += Entry(version, checksum, groups, totalUncompressedLength, digest) } val end = buf.readerIndex() diff --git a/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt b/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt index 2bef1a4d..c629fe26 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt @@ -3,5 +3,6 @@ package org.openrs2.cache public enum class MasterIndexFormat { ORIGINAL, VERSIONED, - WHIRLPOOL + WHIRLPOOL, + LENGTHS } diff --git a/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt b/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt index d7e5ad55..f51cdad4 100644 --- a/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt +++ b/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt @@ -5,6 +5,7 @@ import io.netty.buffer.ByteBufUtil import io.netty.buffer.Unpooled import org.openrs2.buffer.use import org.openrs2.crypto.Rsa +import org.openrs2.crypto.Whirlpool import java.nio.file.Path import kotlin.test.Test import kotlin.test.assertEquals @@ -22,7 +23,7 @@ class Js5MasterIndexTest { MasterIndexFormat.ORIGINAL, mutableListOf( Js5MasterIndex.Entry( - 0, 609698396, ByteBufUtil.decodeHexDump( + 0, 609698396, 0, 0, ByteBufUtil.decodeHexDump( "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" ) @@ -43,28 +44,28 @@ class Js5MasterIndexTest { MasterIndexFormat.VERSIONED, mutableListOf( Js5MasterIndex.Entry( - 0, 609698396, ByteBufUtil.decodeHexDump( + 0, 609698396, 0, 0, ByteBufUtil.decodeHexDump( "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" ) ), Js5MasterIndex.Entry( - 0x12345678, 78747481, ByteBufUtil.decodeHexDump( + 0x12345678, 78747481, 0, 0, ByteBufUtil.decodeHexDump( "180ff4ad371f56d4a90d81e0b69b23836cd9b101b828f18b7e6d232c4d302539" + "638eb2e9259957645aae294f09b2d669c93dbbfc0d8359f1b232ae468f678ca1" ) ), - Js5MasterIndex.Entry(0, 0, null), + Js5MasterIndex.Entry(0, 0, 0, 0, null), Js5MasterIndex.Entry( - 0x9ABCDEF0.toInt(), -456081154, ByteBufUtil.decodeHexDump( + 0x9ABCDEF0.toInt(), -456081154, 0, 0, ByteBufUtil.decodeHexDump( "972003261b7628525346e0052567662e5695147ad710f877b63b9ab53b3f6650" + "ca003035fde4398b2ef73a60e4b13798aa597a30c1bf0a13c0cd412394af5f96" ) ), - Js5MasterIndex.Entry(0, 0, null), - Js5MasterIndex.Entry(0, 0, null), + Js5MasterIndex.Entry(0, 0, 0, 0, null), + Js5MasterIndex.Entry(0, 0, 0, 0, null), Js5MasterIndex.Entry( - 0xAA55AA55.toInt(), 186613982, ByteBufUtil.decodeHexDump( + 0xAA55AA55.toInt(), 186613982, 0, 0, ByteBufUtil.decodeHexDump( "d50a6e9abd3b5269606304dc2769cbc8618e1ae6ff705291c0dfcc374e450dd2" + "5f1be5f1d5459651d22d3e87ef0a1c69be7807f661cd001be24a6609f6d57916" ) @@ -85,13 +86,13 @@ class Js5MasterIndexTest { MasterIndexFormat.WHIRLPOOL, mutableListOf( Js5MasterIndex.Entry( - 0, 668177970, ByteBufUtil.decodeHexDump( + 0, 668177970, 0, 0, ByteBufUtil.decodeHexDump( "2faa83116e1d1719d5db15f128eb57f62afbf0207c47bced3f558ec17645d138" + "72f4fb9b0e36a5f6f5d30e1295b3fa49556dfd0819cb5137f3b69f64155f3fb7" ) ), Js5MasterIndex.Entry( - 0, 1925442845, ByteBufUtil.decodeHexDump( + 0, 1925442845, 0, 0, ByteBufUtil.decodeHexDump( "fcc45b0ab6d0067889e44de0004bcbb6cc538aff8f80edf1b49b583cedd73fea" + "937ae6990235257fe8aa35c44d35450c13e670711337ee5116957cd98cc27985" ) @@ -101,6 +102,27 @@ class Js5MasterIndexTest { ) } + @Test + fun testCreateLengths() { + val index = Store.open(ROOT.resolve("lengths")).use { store -> + Js5MasterIndex.create(store) + } + + assertEquals( + Js5MasterIndex( + MasterIndexFormat.LENGTHS, + mutableListOf( + Js5MasterIndex.Entry( + 0x12345678, -1080883457, 3, 123, ByteBufUtil.decodeHexDump( + "0bf30b80b7213154ada5c3797be15a8fbb6a96a80432e2093e10617bcb4e67de" + + "9a858211cabe844c6fa3a1fbfe3164a3e4e1918983c69597dff3fc3c53096884" + ) + ) + ) + ), index + ) + } + @Test fun testReadOriginal() { Unpooled.wrappedBuffer(encodedOriginal).use { buf -> @@ -242,6 +264,25 @@ class Js5MasterIndexTest { } } + @Test + fun testReadLengths() { + Unpooled.wrappedBuffer(encodedLengths).use { buf -> + val index = Js5MasterIndex.read(buf, MasterIndexFormat.LENGTHS) + assertEquals(decodedLengths, index) + } + } + + @Test + fun testWriteLengths() { + ByteBufAllocator.DEFAULT.buffer().use { actual -> + decodedLengths.write(actual) + + Unpooled.wrappedBuffer(encodedLengths).use { expected -> + assertEquals(expected, actual) + } + } + } + private companion object { private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI()) private val PRIVATE_KEY = Rsa.readPrivateKey(ROOT.resolve("private.key")) @@ -251,9 +292,9 @@ class Js5MasterIndexTest { private val decodedOriginal = Js5MasterIndex( MasterIndexFormat.ORIGINAL, mutableListOf( - Js5MasterIndex.Entry(0, 1, null), - Js5MasterIndex.Entry(0, 3, null), - Js5MasterIndex.Entry(0, 5, null) + Js5MasterIndex.Entry(0, 1, 0, 0, null), + Js5MasterIndex.Entry(0, 3, 0, 0, null), + Js5MasterIndex.Entry(0, 5, 0, 0, null) ) ) @@ -261,9 +302,9 @@ class Js5MasterIndexTest { private val decodedVersioned = Js5MasterIndex( MasterIndexFormat.VERSIONED, mutableListOf( - Js5MasterIndex.Entry(0, 1, null), - Js5MasterIndex.Entry(2, 3, null), - Js5MasterIndex.Entry(4, 5, null) + Js5MasterIndex.Entry(0, 1, 0, 0, null), + Js5MasterIndex.Entry(2, 3, 0, 0, null), + Js5MasterIndex.Entry(4, 5, 0, 0, null) ) ) @@ -281,7 +322,7 @@ class Js5MasterIndexTest { MasterIndexFormat.WHIRLPOOL, mutableListOf( Js5MasterIndex.Entry( - 0x01234567, 0x89ABCDEF.toInt(), ByteBufUtil.decodeHexDump( + 0x01234567, 0x89ABCDEF.toInt(), 0, 0, ByteBufUtil.decodeHexDump( "0e1a2b93c80a41c7ad2a985dff707a6a8ff82e229cbc468f04191198920955a1" + "4b3d7eab77a17faf99208dee5b44afb789962ad79f230b3b59106a0af892219c" ) @@ -302,7 +343,7 @@ class Js5MasterIndexTest { private val decodedWhirlpoolNullDigest = Js5MasterIndex( MasterIndexFormat.WHIRLPOOL, mutableListOf( - Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), null) + Js5MasterIndex.Entry(0x01234567, 0x89ABCDEF.toInt(), 0, 0, null) ) ) @@ -321,5 +362,24 @@ class Js5MasterIndexTest { "57707ce3f4f5a7af8471eda5c0c0748454a9cbb48c25ebe4e7fd94e3881b6461" + "d06e2bce128dc96decb537b8e9611591d445d7dfd3701d25ac05f8d091581aef" ) + + private val decodedLengths = Js5MasterIndex( + MasterIndexFormat.LENGTHS, + mutableListOf( + Js5MasterIndex.Entry(0x012345678, 0x89ABCDEF.toInt(), 3, 123, ByteArray(Whirlpool.DIGESTBYTES)) + ) + ) + private val encodedLengths = ByteBufUtil.decodeHexDump( + "01" + + "89abcdef" + + "12345678" + + "00000003" + + "0000007b" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000" + + "0a" + + "3d9f704a2b2e1b6f4e7b3a9b558baed7ccb10787a754b6fd36acb77ba3491726" + + "fef29e470218a98693bfc1b98a611f15e0b35a11bd181830ff4912377653a87a" + ) } } diff --git a/cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat b/cache/src/test/resources/org/openrs2/cache/master-index/lengths/255/0.dat new file mode 100644 index 0000000000000000000000000000000000000000..f0f35870f4bb70caad891d010ca630ed9d05368b GIT binary patch literal 85 zcmZQzU|_U}0bevKbi|R{_CpAUFdAKoKJ#P61*rAQk~)Mj(c$WME(dvzdW3 KNEE0SA`bwg#|r-d literal 0 HcmV?d00001