From 4126abfaad9cb91a2bc0c22b9eecce955d063e2e Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 6 Feb 2021 20:39:29 +0000 Subject: [PATCH] Add support for unversioned master indexes Signed-off-by: Graham --- .../openrs2/archive/cache/CacheImporter.kt | 11 ++-- .../openrs2/archive/cache/ImportCommand.kt | 5 +- .../archive/cache/ImportMasterIndexCommand.kt | 5 +- .../archive/cache/Js5ChannelHandler.kt | 4 +- .../org/openrs2/cache/Js5MasterIndex.kt | 35 ++++++++++-- .../org/openrs2/cache/MasterIndexFormat.kt | 6 +++ .../org/openrs2/cache/Js5MasterIndexTest.kt | 53 +++++++++++++++---- 7 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index 7b4f4aca8b..9b487c6446 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -12,6 +12,7 @@ import org.openrs2.cache.Js5Compression import org.openrs2.cache.Js5CompressionType import org.openrs2.cache.Js5Index import org.openrs2.cache.Js5MasterIndex +import org.openrs2.cache.MasterIndexFormat import org.openrs2.cache.Store import org.openrs2.cache.VersionTrailer import org.openrs2.crypto.Whirlpool @@ -63,6 +64,7 @@ public class CacheImporter @Inject constructor( public suspend fun import( store: Store, + masterIndexFormat: MasterIndexFormat?, game: String, build: Int?, timestamp: Instant?, @@ -75,7 +77,7 @@ public class CacheImporter @Inject constructor( val gameId = getGameId(connection, game) // import master index - val masterIndex = createMasterIndex(store) + val masterIndex = createMasterIndex(store, masterIndexFormat) try { addMasterIndex(connection, masterIndex, gameId, build, timestamp, name, description, false) } finally { @@ -133,6 +135,7 @@ public class CacheImporter @Inject constructor( public suspend fun importMasterIndex( buf: ByteBuf, + format: MasterIndexFormat, game: String, build: Int?, timestamp: Instant?, @@ -140,7 +143,7 @@ public class CacheImporter @Inject constructor( description: String? ) { Js5Compression.uncompress(buf.slice()).use { uncompressed -> - val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf) + val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice(), format), buf) database.execute { connection -> prepare(connection) @@ -311,11 +314,11 @@ public class CacheImporter @Inject constructor( } } - private fun createMasterIndex(store: Store): MasterIndex { + private fun createMasterIndex(store: Store, format: MasterIndexFormat?): MasterIndex { val index = Js5MasterIndex.create(store) alloc.buffer().use { uncompressed -> - index.write(uncompressed) + index.write(uncompressed, format ?: index.minimumFormat) Js5Compression.compress(uncompressed, Js5CompressionType.UNCOMPRESSED).use { buf -> return MasterIndex(index, buf.retain()) diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt index dacd90b2ae..1787215de0 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt @@ -3,15 +3,18 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.path import com.google.inject.Guice import kotlinx.coroutines.runBlocking import org.openrs2.archive.ArchiveModule +import org.openrs2.cache.MasterIndexFormat import org.openrs2.cache.Store import org.openrs2.cli.instant public class ImportCommand : CliktCommand(name = "import") { + private val masterIndexFormat by option().enum() private val build by option().int() private val timestamp by option().instant() private val name by option() @@ -29,7 +32,7 @@ public class ImportCommand : CliktCommand(name = "import") { val importer = injector.getInstance(CacheImporter::class.java) Store.open(input).use { store -> - importer.import(store, game, build, timestamp, name, description) + importer.import(store, masterIndexFormat, game, build, timestamp, name, description) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt index 0577d554b7..ef866fd67e 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt @@ -3,6 +3,7 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.path import com.google.inject.Guice @@ -10,6 +11,7 @@ import io.netty.buffer.Unpooled import kotlinx.coroutines.runBlocking import org.openrs2.archive.ArchiveModule import org.openrs2.buffer.use +import org.openrs2.cache.MasterIndexFormat import org.openrs2.cli.instant import java.nio.file.Files @@ -20,6 +22,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index private val description by option() private val game by argument() + private val format by argument().enum() private val input by argument().path( mustExist = true, canBeDir = false, @@ -31,7 +34,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index val importer = injector.getInstance(CacheImporter::class.java) Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> - importer.importMasterIndex(buf, game, build, timestamp, name, description) + importer.importMasterIndex(buf, format, game, build, timestamp, name, description) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt index b57999e421..9ce31c4a55 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt @@ -13,6 +13,7 @@ import org.openrs2.cache.Js5Archive import org.openrs2.cache.Js5Compression import org.openrs2.cache.Js5Index import org.openrs2.cache.Js5MasterIndex +import org.openrs2.cache.MasterIndexFormat import org.openrs2.protocol.Rs2Decoder import org.openrs2.protocol.Rs2Encoder import org.openrs2.protocol.js5.Js5Request @@ -36,6 +37,7 @@ public class Js5ChannelHandler( private var build: Int, private val continuation: Continuation, private val importer: CacheImporter, + private val masterIndexFormat: MasterIndexFormat = MasterIndexFormat.VERSIONED, private val maxInFlightRequests: Int = 200, maxBuildAttempts: Int = 10 ) : SimpleChannelInboundHandler(Object::class.java) { @@ -159,7 +161,7 @@ public class Js5ChannelHandler( private fun processMasterIndex(buf: ByteBuf) { masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed -> - Js5MasterIndex.read(uncompressed) + Js5MasterIndex.read(uncompressed, masterIndexFormat) } val rawIndexes = runBlocking { diff --git a/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt b/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt index 30a0b09b6b..7df7425592 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt @@ -7,10 +7,24 @@ import org.openrs2.buffer.use public inline class Js5MasterIndex(public val entries: MutableList = mutableListOf()) { public data class Entry(public var version: Int, public var checksum: Int) - public fun write(buf: ByteBuf) { + public val minimumFormat: MasterIndexFormat + get() { + for (entry in entries) { + if (entry.version != 0) { + return MasterIndexFormat.VERSIONED + } + } + + return MasterIndexFormat.ORIGINAL + } + + public fun write(buf: ByteBuf, format: MasterIndexFormat) { for (entry in entries) { buf.writeInt(entry.checksum) - buf.writeInt(entry.version) + + if (format >= MasterIndexFormat.VERSIONED) { + buf.writeInt(entry.version) + } } } @@ -45,13 +59,24 @@ public inline class Js5MasterIndex(public val entries: MutableList = muta return index } - public fun read(buf: ByteBuf): Js5MasterIndex { - require(buf.readableBytes() % 8 == 0) + public fun read(buf: ByteBuf, format: MasterIndexFormat): Js5MasterIndex { + when (format) { + MasterIndexFormat.ORIGINAL -> { + require(buf.readableBytes() % 4 == 0) + } + MasterIndexFormat.VERSIONED -> { + require(buf.readableBytes() % 8 == 0) + } + } val index = Js5MasterIndex() while (buf.isReadable) { val checksum = buf.readInt() - val version = buf.readInt() + val version = if (format >= MasterIndexFormat.VERSIONED) { + buf.readInt() + } else { + 0 + } index.entries += Entry(version, checksum) } return index diff --git a/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt b/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt new file mode 100644 index 0000000000..7a11e0a452 --- /dev/null +++ b/cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt @@ -0,0 +1,6 @@ +package org.openrs2.cache + +public enum class MasterIndexFormat { + ORIGINAL, + VERSIONED +} diff --git a/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt b/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt index 10020b82b5..f3231730b5 100644 --- a/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt +++ b/cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt @@ -9,8 +9,18 @@ import kotlin.test.assertEquals object Js5MasterIndexTest { private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI()) - private val encoded = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4) - private val decoded = Js5MasterIndex( + + private val encodedOriginal = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5) + private val decodedOriginal = Js5MasterIndex( + mutableListOf( + Js5MasterIndex.Entry(0, 1), + Js5MasterIndex.Entry(0, 3), + Js5MasterIndex.Entry(0, 5) + ) + ) + + private val encodedVersioned = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4) + private val decodedVersioned = Js5MasterIndex( mutableListOf( Js5MasterIndex.Entry(0, 1), Js5MasterIndex.Entry(2, 3), @@ -18,6 +28,12 @@ object Js5MasterIndexTest { ) ) + @Test + fun testMinimumFormat() { + assertEquals(MasterIndexFormat.ORIGINAL, decodedOriginal.minimumFormat) + assertEquals(MasterIndexFormat.VERSIONED, decodedVersioned.minimumFormat) + } + @Test fun testCreate() { val index = Store.open(ROOT).use { store -> @@ -40,19 +56,38 @@ object Js5MasterIndexTest { } @Test - fun testRead() { - Unpooled.wrappedBuffer(encoded).use { buf -> - val index = Js5MasterIndex.read(buf) - assertEquals(decoded, index) + fun testReadOriginal() { + Unpooled.wrappedBuffer(encodedOriginal).use { buf -> + val index = Js5MasterIndex.read(buf, MasterIndexFormat.ORIGINAL) + assertEquals(decodedOriginal, index) + } + } + + @Test + fun testWriteOriginal() { + ByteBufAllocator.DEFAULT.buffer().use { actual -> + decodedOriginal.write(actual, MasterIndexFormat.ORIGINAL) + + Unpooled.wrappedBuffer(encodedOriginal).use { expected -> + assertEquals(expected, actual) + } + } + } + + @Test + fun testReadVersioned() { + Unpooled.wrappedBuffer(encodedVersioned).use { buf -> + val index = Js5MasterIndex.read(buf, MasterIndexFormat.VERSIONED) + assertEquals(decodedVersioned, index) } } @Test - fun testWrite() { + fun testWriteVersioned() { ByteBufAllocator.DEFAULT.buffer().use { actual -> - decoded.write(actual) + decodedVersioned.write(actual, MasterIndexFormat.VERSIONED) - Unpooled.wrappedBuffer(encoded).use { expected -> + Unpooled.wrappedBuffer(encodedVersioned).use { expected -> assertEquals(expected, actual) } }