Add support for unversioned master indexes

Signed-off-by: Graham <gpe@openrs2.org>

archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt

@@ -12,6 +12,7 @@ import org.openrs2.cache.Js5Compression
 import org.openrs2.cache.Js5CompressionType
 import org.openrs2.cache.Js5Index
 import org.openrs2.cache.Js5MasterIndex
+import org.openrs2.cache.MasterIndexFormat
 import org.openrs2.cache.Store
 import org.openrs2.cache.VersionTrailer
 import org.openrs2.crypto.Whirlpool
@@ -63,6 +64,7 @@ public class CacheImporter @Inject constructor(
 
     public suspend fun import(
         store: Store,
+        masterIndexFormat: MasterIndexFormat?,
         game: String,
         build: Int?,
         timestamp: Instant?,
@@ -75,7 +77,7 @@ public class CacheImporter @Inject constructor(
             val gameId = getGameId(connection, game)
 
             // import master index
-            val masterIndex = createMasterIndex(store)
+            val masterIndex = createMasterIndex(store, masterIndexFormat)
             try {
                 addMasterIndex(connection, masterIndex, gameId, build, timestamp, name, description, false)
             } finally {
@@ -133,6 +135,7 @@ public class CacheImporter @Inject constructor(
 
     public suspend fun importMasterIndex(
         buf: ByteBuf,
+        format: MasterIndexFormat,
         game: String,
         build: Int?,
         timestamp: Instant?,
@@ -140,7 +143,7 @@ public class CacheImporter @Inject constructor(
         description: String?
     ) {
         Js5Compression.uncompress(buf.slice()).use { uncompressed ->
-            val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf)
+            val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice(), format), buf)
 
             database.execute { connection ->
                 prepare(connection)
@@ -311,11 +314,11 @@ public class CacheImporter @Inject constructor(
         }
     }
 
-    private fun createMasterIndex(store: Store): MasterIndex {
+    private fun createMasterIndex(store: Store, format: MasterIndexFormat?): MasterIndex {
         val index = Js5MasterIndex.create(store)
 
         alloc.buffer().use { uncompressed ->
-            index.write(uncompressed)
+            index.write(uncompressed, format ?: index.minimumFormat)
 
             Js5Compression.compress(uncompressed, Js5CompressionType.UNCOMPRESSED).use { buf ->
                 return MasterIndex(index, buf.retain())

archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt

@@ -3,15 +3,18 @@ package org.openrs2.archive.cache
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
 import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.path
 import com.google.inject.Guice
 import kotlinx.coroutines.runBlocking
 import org.openrs2.archive.ArchiveModule
+import org.openrs2.cache.MasterIndexFormat
 import org.openrs2.cache.Store
 import org.openrs2.cli.instant
 
 public class ImportCommand : CliktCommand(name = "import") {
+    private val masterIndexFormat by option().enum<MasterIndexFormat>()
     private val build by option().int()
     private val timestamp by option().instant()
     private val name by option()
@@ -29,7 +32,7 @@ public class ImportCommand : CliktCommand(name = "import") {
         val importer = injector.getInstance(CacheImporter::class.java)
 
         Store.open(input).use { store ->
-            importer.import(store, game, build, timestamp, name, description)
+            importer.import(store, masterIndexFormat, game, build, timestamp, name, description)
         }
     }
 }

archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt

@@ -3,6 +3,7 @@ package org.openrs2.archive.cache
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
 import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.path
 import com.google.inject.Guice
@@ -10,6 +11,7 @@ import io.netty.buffer.Unpooled
 import kotlinx.coroutines.runBlocking
 import org.openrs2.archive.ArchiveModule
 import org.openrs2.buffer.use
+import org.openrs2.cache.MasterIndexFormat
 import org.openrs2.cli.instant
 import java.nio.file.Files
 
@@ -20,6 +22,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
     private val description by option()
 
     private val game by argument()
+    private val format by argument().enum<MasterIndexFormat>()
     private val input by argument().path(
         mustExist = true,
         canBeDir = false,
@@ -31,7 +34,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
         val importer = injector.getInstance(CacheImporter::class.java)
 
         Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
-            importer.importMasterIndex(buf, game, build, timestamp, name, description)
+            importer.importMasterIndex(buf, format, game, build, timestamp, name, description)
         }
     }
 }

archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt

@@ -13,6 +13,7 @@ import org.openrs2.cache.Js5Archive
 import org.openrs2.cache.Js5Compression
 import org.openrs2.cache.Js5Index
 import org.openrs2.cache.Js5MasterIndex
+import org.openrs2.cache.MasterIndexFormat
 import org.openrs2.protocol.Rs2Decoder
 import org.openrs2.protocol.Rs2Encoder
 import org.openrs2.protocol.js5.Js5Request
@@ -36,6 +37,7 @@ public class Js5ChannelHandler(
     private var build: Int,
     private val continuation: Continuation<Unit>,
     private val importer: CacheImporter,
+    private val masterIndexFormat: MasterIndexFormat = MasterIndexFormat.VERSIONED,
     private val maxInFlightRequests: Int = 200,
     maxBuildAttempts: Int = 10
 ) : SimpleChannelInboundHandler<Any>(Object::class.java) {
@@ -159,7 +161,7 @@ public class Js5ChannelHandler(
 
     private fun processMasterIndex(buf: ByteBuf) {
         masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed ->
-            Js5MasterIndex.read(uncompressed)
+            Js5MasterIndex.read(uncompressed, masterIndexFormat)
         }
 
         val rawIndexes = runBlocking {

cache/src/main/kotlin/org/openrs2/cache/Js5MasterIndex.kt

@@ -7,10 +7,24 @@ import org.openrs2.buffer.use
 public inline class Js5MasterIndex(public val entries: MutableList<Entry> = mutableListOf()) {
     public data class Entry(public var version: Int, public var checksum: Int)
 
-    public fun write(buf: ByteBuf) {
+    public val minimumFormat: MasterIndexFormat
+        get() {
+            for (entry in entries) {
+                if (entry.version != 0) {
+                    return MasterIndexFormat.VERSIONED
+                }
+            }
+
+            return MasterIndexFormat.ORIGINAL
+        }
+
+    public fun write(buf: ByteBuf, format: MasterIndexFormat) {
         for (entry in entries) {
             buf.writeInt(entry.checksum)
-            buf.writeInt(entry.version)
+
+            if (format >= MasterIndexFormat.VERSIONED) {
+                buf.writeInt(entry.version)
+            }
         }
     }
 
@@ -45,13 +59,24 @@ public inline class Js5MasterIndex(public val entries: MutableList<Entry> = muta
             return index
         }
 
-        public fun read(buf: ByteBuf): Js5MasterIndex {
+        public fun read(buf: ByteBuf, format: MasterIndexFormat): Js5MasterIndex {
-            require(buf.readableBytes() % 8 == 0)
+            when (format) {
+                MasterIndexFormat.ORIGINAL -> {
+                    require(buf.readableBytes() % 4 == 0)
+                }
+                MasterIndexFormat.VERSIONED -> {
+                    require(buf.readableBytes() % 8 == 0)
+                }
+            }
 
             val index = Js5MasterIndex()
             while (buf.isReadable) {
                 val checksum = buf.readInt()
-                val version = buf.readInt()
+                val version = if (format >= MasterIndexFormat.VERSIONED) {
+                    buf.readInt()
+                } else {
+                    0
+                }
                 index.entries += Entry(version, checksum)
             }
             return index

cache/src/main/kotlin/org/openrs2/cache/MasterIndexFormat.kt

@@ -0,0 +1,6 @@
+package org.openrs2.cache
+
+public enum class MasterIndexFormat {
+    ORIGINAL,
+    VERSIONED
+}

cache/src/test/kotlin/org/openrs2/cache/Js5MasterIndexTest.kt

@@ -9,8 +9,18 @@ import kotlin.test.assertEquals
 
 object Js5MasterIndexTest {
     private val ROOT = Path.of(FlatFileStoreTest::class.java.getResource("master-index").toURI())
-    private val encoded = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
+
-    private val decoded = Js5MasterIndex(
+    private val encodedOriginal = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 5)
+    private val decodedOriginal = Js5MasterIndex(
+        mutableListOf(
+            Js5MasterIndex.Entry(0, 1),
+            Js5MasterIndex.Entry(0, 3),
+            Js5MasterIndex.Entry(0, 5)
+        )
+    )
+
+    private val encodedVersioned = byteArrayOf(0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, 4)
+    private val decodedVersioned = Js5MasterIndex(
         mutableListOf(
             Js5MasterIndex.Entry(0, 1),
             Js5MasterIndex.Entry(2, 3),
@@ -18,6 +28,12 @@ object Js5MasterIndexTest {
         )
     )
 
+    @Test
+    fun testMinimumFormat() {
+        assertEquals(MasterIndexFormat.ORIGINAL, decodedOriginal.minimumFormat)
+        assertEquals(MasterIndexFormat.VERSIONED, decodedVersioned.minimumFormat)
+    }
+
     @Test
     fun testCreate() {
         val index = Store.open(ROOT).use { store ->
@@ -40,19 +56,38 @@ object Js5MasterIndexTest {
     }
 
     @Test
-    fun testRead() {
+    fun testReadOriginal() {
-        Unpooled.wrappedBuffer(encoded).use { buf ->
+        Unpooled.wrappedBuffer(encodedOriginal).use { buf ->
-            val index = Js5MasterIndex.read(buf)
+            val index = Js5MasterIndex.read(buf, MasterIndexFormat.ORIGINAL)
-            assertEquals(decoded, index)
+            assertEquals(decodedOriginal, index)
+        }
+    }
+
+    @Test
+    fun testWriteOriginal() {
+        ByteBufAllocator.DEFAULT.buffer().use { actual ->
+            decodedOriginal.write(actual, MasterIndexFormat.ORIGINAL)
+
+            Unpooled.wrappedBuffer(encodedOriginal).use { expected ->
+                assertEquals(expected, actual)
+            }
+        }
+    }
+
+    @Test
+    fun testReadVersioned() {
+        Unpooled.wrappedBuffer(encodedVersioned).use { buf ->
+            val index = Js5MasterIndex.read(buf, MasterIndexFormat.VERSIONED)
+            assertEquals(decodedVersioned, index)
         }
     }
 
     @Test
-    fun testWrite() {
+    fun testWriteVersioned() {
         ByteBufAllocator.DEFAULT.buffer().use { actual ->
-            decoded.write(actual)
+            decodedVersioned.write(actual, MasterIndexFormat.VERSIONED)
 
-            Unpooled.wrappedBuffer(encoded).use { expected ->
+            Unpooled.wrappedBuffer(encodedVersioned).use { expected ->
                 assertEquals(expected, actual)
             }
         }