Add support for unversioned master indexes

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent 4154e4fdb4
commit 4126abfaad
  1. 11
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  2. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt
  3. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt
  4. 4
      archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt
  5. 35
      cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt
  6. 6
      cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt
  7. 53
      cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt

@ -12,6 +12,7 @@ import org.openrs2.cache.Js5Compression
import org.openrs2.cache.Js5CompressionType
import org.openrs2.cache.Js5Index
import org.openrs2.cache.Js5MasterIndex
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cache.Store
import org.openrs2.cache.VersionTrailer
import org.openrs2.crypto.Whirlpool
@ -63,6 +64,7 @@ public class CacheImporter @Inject constructor(
public suspend fun import(
store: Store,
masterIndexFormat: MasterIndexFormat?,
game: String,
build: Int?,
timestamp: Instant?,
@ -75,7 +77,7 @@ public class CacheImporter @Inject constructor(
val gameId = getGameId(connection, game)
// import master index
val masterIndex = createMasterIndex(store)
val masterIndex = createMasterIndex(store, masterIndexFormat)
try {
addMasterIndex(connection, masterIndex, gameId, build, timestamp, name, description, false)
} finally {
@ -133,6 +135,7 @@ public class CacheImporter @Inject constructor(
public suspend fun importMasterIndex(
buf: ByteBuf,
format: MasterIndexFormat,
game: String,
build: Int?,
timestamp: Instant?,
@ -140,7 +143,7 @@ public class CacheImporter @Inject constructor(
description: String?
) {
Js5Compression.uncompress(buf.slice()).use { uncompressed ->
val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf)
val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice(), format), buf)
database.execute { connection ->
prepare(connection)
@ -311,11 +314,11 @@ public class CacheImporter @Inject constructor(
}
}
private fun createMasterIndex(store: Store): MasterIndex {
private fun createMasterIndex(store: Store, format: MasterIndexFormat?): MasterIndex {
val index = Js5MasterIndex.create(store)
alloc.buffer().use { uncompressed ->
index.write(uncompressed)
index.write(uncompressed, format ?: index.minimumFormat)
Js5Compression.compress(uncompressed, Js5CompressionType.UNCOMPRESSED).use { buf ->
return MasterIndex(index, buf.retain())

@ -3,15 +3,18 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice
import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cache.Store
import org.openrs2.cli.instant
public class ImportCommand : CliktCommand(name = "import") {
private val masterIndexFormat by option().enum<MasterIndexFormat>()
private val build by option().int()
private val timestamp by option().instant()
private val name by option()
@ -29,7 +32,7 @@ public class ImportCommand : CliktCommand(name = "import") {
val importer = injector.getInstance(CacheImporter::class.java)
Store.open(input).use { store ->
importer.import(store, game, build, timestamp, name, description)
importer.import(store, masterIndexFormat, game, build, timestamp, name, description)
}
}
}

@ -3,6 +3,7 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice
@ -10,6 +11,7 @@ import io.netty.buffer.Unpooled
import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule
import org.openrs2.buffer.use
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.cli.instant
import java.nio.file.Files
@ -20,6 +22,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
private val description by option()
private val game by argument()
private val format by argument().enum<MasterIndexFormat>()
private val input by argument().path(
mustExist = true,
canBeDir = false,
@ -31,7 +34,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
val importer = injector.getInstance(CacheImporter::class.java)
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
importer.importMasterIndex(buf, game, build, timestamp, name, description)
importer.importMasterIndex(buf, format, game, build, timestamp, name, description)
}
}
}

@ -13,6 +13,7 @@ import org.openrs2.cache.Js5Archive
import org.openrs2.cache.Js5Compression
import org.openrs2.cache.Js5Index
import org.openrs2.cache.Js5MasterIndex
import org.openrs2.cache.MasterIndexFormat
import org.openrs2.protocol.Rs2Decoder
import org.openrs2.protocol.Rs2Encoder
import org.openrs2.protocol.js5.Js5Request
@ -36,6 +37,7 @@ public class Js5ChannelHandler(
private var build: Int,
private val continuation: Continuation<Unit>,
private val importer: CacheImporter,
private val masterIndexFormat: MasterIndexFormat = MasterIndexFormat.VERSIONED,
private val maxInFlightRequests: Int = 200,
maxBuildAttempts: Int = 10
) : SimpleChannelInboundHandler<Any>(Object::class.java) {
@ -159,7 +161,7 @@ public class Js5ChannelHandler(
private fun processMasterIndex(buf: ByteBuf) {
masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed ->
Js5MasterIndex.read(uncompressed)
Js5MasterIndex.read(uncompressed, masterIndexFormat)
}
val rawIndexes = runBlocking {

@ -7,10 +7,24 @@ import org.openrs2.buffer.use
public inline class Js5MasterIndex(public val entries: MutableList<Entry> = mutableListOf()) {
public data class Entry(public var version: Int, public var checksum: Int)
public fun write(buf: ByteBuf) {
public val minimumFormat: MasterIndexFormat
get() {
for (entry in entries) {
if (entry.version != 0) {
return MasterIndexFormat.VERSIONED
}
}
return MasterIndexFormat.ORIGINAL
}
public fun write(buf: ByteBuf, format: MasterIndexFormat) {
for (entry in entries) {
buf.writeInt(entry.checksum)
buf.writeInt(entry.version)
if (format >= MasterIndexFormat.VERSIONED) {
buf.writeInt(entry.version)
}
}
}
@ -45,13 +59,24 @@ public inline class Js5MasterIndex(public val entries: MutableList<Entry> = muta
return index
}
public fun read(buf: ByteBuf): Js5MasterIndex {
require(buf.readableBytes() % 8 == 0)
public fun read(buf: ByteBuf, format: MasterIndexFormat): Js5MasterIndex {
when (format) {
MasterIndexFormat.ORIGINAL -> {
require(buf.readableBytes() % 4 == 0)
}
MasterIndexFormat.VERSIONED -> {
require(buf.readableBytes() % 8 == 0)
}
}
val index = Js5MasterIndex()
while (buf.isReadable) {
val checksum = buf.readInt()
val version = buf.readInt()
val version = if (format >= MasterIndexFormat.VERSIONED) {
buf.readInt()
} else {
0
}
index.entries += Entry(version, checksum)
}
return index

@ -0,0 +1,6 @@
package org.openrs2.cache
public enum class MasterIndexFormat {
ORIGINAL,
VERSIONED
}

@ -9,8 +9,18 @@ import kotlin.test.assertEquals
object Js5MasterIndexTest {
private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI())
private val encoded = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
private val decoded = Js5MasterIndex(
private val encodedOriginal = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5)
private val decodedOriginal = Js5MasterIndex(
mutableListOf(
Js5MasterIndex.Entry(0, 1),
Js5MasterIndex.Entry(0, 3),
Js5MasterIndex.Entry(0, 5)
)
)
private val encodedVersioned = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
private val decodedVersioned = Js5MasterIndex(
mutableListOf(
Js5MasterIndex.Entry(0, 1),
Js5MasterIndex.Entry(2, 3),
@ -18,6 +28,12 @@ object Js5MasterIndexTest {
)
)
@Test
fun testMinimumFormat() {
assertEquals(MasterIndexFormat.ORIGINAL, decodedOriginal.minimumFormat)
assertEquals(MasterIndexFormat.VERSIONED, decodedVersioned.minimumFormat)
}
@Test
fun testCreate() {
val index = Store.open(ROOT).use { store ->
@ -40,19 +56,38 @@ object Js5MasterIndexTest {
}
@Test
fun testRead() {
Unpooled.wrappedBuffer(encoded).use { buf ->
val index = Js5MasterIndex.read(buf)
assertEquals(decoded, index)
fun testReadOriginal() {
Unpooled.wrappedBuffer(encodedOriginal).use { buf ->
val index = Js5MasterIndex.read(buf, MasterIndexFormat.ORIGINAL)
assertEquals(decodedOriginal, index)
}
}
@Test
fun testWriteOriginal() {
ByteBufAllocator.DEFAULT.buffer().use { actual ->
decodedOriginal.write(actual, MasterIndexFormat.ORIGINAL)
Unpooled.wrappedBuffer(encodedOriginal).use { expected ->
assertEquals(expected, actual)
}
}
}
@Test
fun testReadVersioned() {
Unpooled.wrappedBuffer(encodedVersioned).use { buf ->
val index = Js5MasterIndex.read(buf, MasterIndexFormat.VERSIONED)
assertEquals(decodedVersioned, index)
}
}
@Test
fun testWrite() {
fun testWriteVersioned() {
ByteBufAllocator.DEFAULT.buffer().use { actual ->
decoded.write(actual)
decodedVersioned.write(actual, MasterIndexFormat.VERSIONED)
Unpooled.wrappedBuffer(encoded).use { expected ->
Unpooled.wrappedBuffer(encodedVersioned).use { expected ->
assertEquals(expected, actual)
}
}

Loading…
Cancel
Save