Add support for unversioned master indexes

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent 4154e4fdb4
commit 4126abfaad
  1. 11
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  2. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt
  3. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt
  4. 4
      archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt
  5. 35
      cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt
  6. 6
      cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt
  7. 53
      cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt

@ -12,6 +12,7 @@ import org.openrs2.cache.Js5Compression
import org.openrs2.cache.Js5CompressionType import org.openrs2.cache.Js5CompressionType
import org.openrs2.cache.Js5Index import org.openrs2.cache.Js5Index
import org.openrs2.cache.Js5MasterIndex import org.openrs2.cache.Js5MasterIndex
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cache.Store import org.openrs2.cache.Store
import org.openrs2.cache.VersionTrailer import org.openrs2.cache.VersionTrailer
import org.openrs2.crypto.Whirlpool import org.openrs2.crypto.Whirlpool
@ -63,6 +64,7 @@ public class CacheImporter @Inject constructor(
public suspend fun import( public suspend fun import(
store: Store, store: Store,
masterIndexFormat: MasterIndexFormat?,
game: String, game: String,
build: Int?, build: Int?,
timestamp: Instant?, timestamp: Instant?,
@ -75,7 +77,7 @@ public class CacheImporter @Inject constructor(
val gameId = getGameId(connection, game) val gameId = getGameId(connection, game)
// import master index // import master index
val masterIndex = createMasterIndex(store) val masterIndex = createMasterIndex(store, masterIndexFormat)
try { try {
addMasterIndex(connection, masterIndex, gameId, build, timestamp, name, description, false) addMasterIndex(connection, masterIndex, gameId, build, timestamp, name, description, false)
} finally { } finally {
@ -133,6 +135,7 @@ public class CacheImporter @Inject constructor(
public suspend fun importMasterIndex( public suspend fun importMasterIndex(
buf: ByteBuf, buf: ByteBuf,
format: MasterIndexFormat,
game: String, game: String,
build: Int?, build: Int?,
timestamp: Instant?, timestamp: Instant?,
@ -140,7 +143,7 @@ public class CacheImporter @Inject constructor(
description: String? description: String?
) { ) {
Js5Compression.uncompress(buf.slice()).use { uncompressed -> Js5Compression.uncompress(buf.slice()).use { uncompressed ->
val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf) val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice(), format), buf)
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
@ -311,11 +314,11 @@ public class CacheImporter @Inject constructor(
} }
} }
private fun createMasterIndex(store: Store): MasterIndex { private fun createMasterIndex(store: Store, format: MasterIndexFormat?): MasterIndex {
val index = Js5MasterIndex.create(store) val index = Js5MasterIndex.create(store)
alloc.buffer().use { uncompressed -> alloc.buffer().use { uncompressed ->
index.write(uncompressed) index.write(uncompressed, format ?: index.minimumFormat)
Js5Compression.compress(uncompressed, Js5CompressionType.UNCOMPRESSED).use { buf -> Js5Compression.compress(uncompressed, Js5CompressionType.UNCOMPRESSED).use { buf ->
return MasterIndex(index, buf.retain()) return MasterIndex(index, buf.retain())

@ -3,15 +3,18 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice import com.google.inject.Guice
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule import org.openrs2.archive.ArchiveModule
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cache.Store import org.openrs2.cache.Store
import org.openrs2.cli.instant import org.openrs2.cli.instant
public class ImportCommand : CliktCommand(name = "import") { public class ImportCommand : CliktCommand(name = "import") {
private val masterIndexFormat by option().enum<MasterIndexFormat>()
private val build by option().int() private val build by option().int()
private val timestamp by option().instant() private val timestamp by option().instant()
private val name by option() private val name by option()
@ -29,7 +32,7 @@ public class ImportCommand : CliktCommand(name = "import") {
val importer = injector.getInstance(CacheImporter::class.java) val importer = injector.getInstance(CacheImporter::class.java)
Store.open(input).use { store -> Store.open(input).use { store ->
importer.import(store, game, build, timestamp, name, description) importer.import(store, masterIndexFormat, game, build, timestamp, name, description)
} }
} }
} }

@ -3,6 +3,7 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice import com.google.inject.Guice
@ -10,6 +11,7 @@ import io.netty.buffer.Unpooled
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule import org.openrs2.archive.ArchiveModule
import org.openrs2.buffer.use import org.openrs2.buffer.use
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cli.instant import org.openrs2.cli.instant
import java.nio.file.Files import java.nio.file.Files
@ -20,6 +22,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
private val description by option() private val description by option()
private val game by argument() private val game by argument()
private val format by argument().enum<MasterIndexFormat>()
private val input by argument().path( private val input by argument().path(
mustExist = true, mustExist = true,
canBeDir = false, canBeDir = false,
@ -31,7 +34,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
val importer = injector.getInstance(CacheImporter::class.java) val importer = injector.getInstance(CacheImporter::class.java)
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
importer.importMasterIndex(buf, game, build, timestamp, name, description) importer.importMasterIndex(buf, format, game, build, timestamp, name, description)
} }
} }
} }

@ -13,6 +13,7 @@ import org.openrs2.cache.Js5Archive
import org.openrs2.cache.Js5Compression import org.openrs2.cache.Js5Compression
import org.openrs2.cache.Js5Index import org.openrs2.cache.Js5Index
import org.openrs2.cache.Js5MasterIndex import org.openrs2.cache.Js5MasterIndex
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.protocol.Rs2Decoder import org.openrs2.protocol.Rs2Decoder
import org.openrs2.protocol.Rs2Encoder import org.openrs2.protocol.Rs2Encoder
import org.openrs2.protocol.js5.Js5Request import org.openrs2.protocol.js5.Js5Request
@ -36,6 +37,7 @@ public class Js5ChannelHandler(
private var build: Int, private var build: Int,
private val continuation: Continuation<Unit>, private val continuation: Continuation<Unit>,
private val importer: CacheImporter, private val importer: CacheImporter,
private val masterIndexFormat: MasterIndexFormat = MasterIndexFormat.VERSIONED,
private val maxInFlightRequests: Int = 200, private val maxInFlightRequests: Int = 200,
maxBuildAttempts: Int = 10 maxBuildAttempts: Int = 10
) : SimpleChannelInboundHandler<Any>(Object::class.java) { ) : SimpleChannelInboundHandler<Any>(Object::class.java) {
@ -159,7 +161,7 @@ public class Js5ChannelHandler(
private fun processMasterIndex(buf: ByteBuf) { private fun processMasterIndex(buf: ByteBuf) {
masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed -> masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed ->
Js5MasterIndex.read(uncompressed) Js5MasterIndex.read(uncompressed, masterIndexFormat)
} }
val rawIndexes = runBlocking { val rawIndexes = runBlocking {

@ -7,10 +7,24 @@ import org.openrs2.buffer.use
public inline class Js5MasterIndex(public val entries: MutableList<Entry> = mutableListOf()) { public inline class Js5MasterIndex(public val entries: MutableList<Entry> = mutableListOf()) {
public data class Entry(public var version: Int, public var checksum: Int) public data class Entry(public var version: Int, public var checksum: Int)
public fun write(buf: ByteBuf) { public val minimumFormat: MasterIndexFormat
get() {
for (entry in entries) {
if (entry.version != 0) {
return MasterIndexFormat.VERSIONED
}
}
return MasterIndexFormat.ORIGINAL
}
public fun write(buf: ByteBuf, format: MasterIndexFormat) {
for (entry in entries) { for (entry in entries) {
buf.writeInt(entry.checksum) buf.writeInt(entry.checksum)
buf.writeInt(entry.version)
if (format >= MasterIndexFormat.VERSIONED) {
buf.writeInt(entry.version)
}
} }
} }
@ -45,13 +59,24 @@ public inline class Js5MasterIndex(public val entries: MutableList<Entry> = muta
return index return index
} }
public fun read(buf: ByteBuf): Js5MasterIndex { public fun read(buf: ByteBuf, format: MasterIndexFormat): Js5MasterIndex {
require(buf.readableBytes() % 8 == 0) when (format) {
MasterIndexFormat.ORIGINAL -> {
require(buf.readableBytes() % 4 == 0)
}
MasterIndexFormat.VERSIONED -> {
require(buf.readableBytes() % 8 == 0)
}
}
val index = Js5MasterIndex() val index = Js5MasterIndex()
while (buf.isReadable) { while (buf.isReadable) {
val checksum = buf.readInt() val checksum = buf.readInt()
val version = buf.readInt() val version = if (format >= MasterIndexFormat.VERSIONED) {
buf.readInt()
} else {
0
}
index.entries += Entry(version, checksum) index.entries += Entry(version, checksum)
} }
return index return index

@ -0,0 +1,6 @@
package org.openrs2.cache
public enum class MasterIndexFormat {
ORIGINAL,
VERSIONED
}

@ -9,8 +9,18 @@ import kotlin.test.assertEquals
object Js5MasterIndexTest { object Js5MasterIndexTest {
private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI()) private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI())
private val encoded = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
private val decoded = Js5MasterIndex( private val encodedOriginal = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5)
private val decodedOriginal = Js5MasterIndex(
mutableListOf(
Js5MasterIndex.Entry(0, 1),
Js5MasterIndex.Entry(0, 3),
Js5MasterIndex.Entry(0, 5)
)
)
private val encodedVersioned = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
private val decodedVersioned = Js5MasterIndex(
mutableListOf( mutableListOf(
Js5MasterIndex.Entry(0, 1), Js5MasterIndex.Entry(0, 1),
Js5MasterIndex.Entry(2, 3), Js5MasterIndex.Entry(2, 3),
@ -18,6 +28,12 @@ object Js5MasterIndexTest {
) )
) )
@Test
fun testMinimumFormat() {
assertEquals(MasterIndexFormat.ORIGINAL, decodedOriginal.minimumFormat)
assertEquals(MasterIndexFormat.VERSIONED, decodedVersioned.minimumFormat)
}
@Test @Test
fun testCreate() { fun testCreate() {
val index = Store.open(ROOT).use { store -> val index = Store.open(ROOT).use { store ->
@ -40,19 +56,38 @@ object Js5MasterIndexTest {
} }
@Test @Test
fun testRead() { fun testReadOriginal() {
Unpooled.wrappedBuffer(encoded).use { buf -> Unpooled.wrappedBuffer(encodedOriginal).use { buf ->
val index = Js5MasterIndex.read(buf) val index = Js5MasterIndex.read(buf, MasterIndexFormat.ORIGINAL)
assertEquals(decoded, index) assertEquals(decodedOriginal, index)
}
}
@Test
fun testWriteOriginal() {
ByteBufAllocator.DEFAULT.buffer().use { actual ->
decodedOriginal.write(actual, MasterIndexFormat.ORIGINAL)
Unpooled.wrappedBuffer(encodedOriginal).use { expected ->
assertEquals(expected, actual)
}
}
}
@Test
fun testReadVersioned() {
Unpooled.wrappedBuffer(encodedVersioned).use { buf ->
val index = Js5MasterIndex.read(buf, MasterIndexFormat.VERSIONED)
assertEquals(decodedVersioned, index)
} }
} }
@Test @Test
fun testWrite() { fun testWriteVersioned() {
ByteBufAllocator.DEFAULT.buffer().use { actual -> ByteBufAllocator.DEFAULT.buffer().use { actual ->
decoded.write(actual) decodedVersioned.write(actual, MasterIndexFormat.VERSIONED)
Unpooled.wrappedBuffer(encoded).use { expected -> Unpooled.wrappedBuffer(encodedVersioned).use { expected ->
assertEquals(expected, actual) assertEquals(expected, actual)
} }
} }

Loading…
Cancel
Save