Add support for resolving files/groups between old and new engine caches

Signed-off-by: Graham <gpe@openrs2.org>

archive/src/main/kotlin/org/openrs2/archive/cache/CacheCommand.kt

@@ -7,6 +7,7 @@ import org.openrs2.archive.cache.finder.ExtractCommand
 public class CacheCommand : NoOpCliktCommand(name = "cache") {
     init {
         subcommands(
+            CrossPollinateCommand(),
             DownloadCommand(),
             ExtractCommand(),
             ImportCommand(),

archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt

@@ -105,9 +105,10 @@ public class CacheImporter @Inject constructor(
         public val version: Int
     ) : Blob(buf)
 
-    private enum class SourceType {
+    internal enum class SourceType {
         DISK,
-        JS5REMOTE
+        JS5REMOTE,
+        CROSS_POLLINATION
     }
 
     public data class MasterIndexResult(
@@ -540,11 +541,11 @@ public class CacheImporter @Inject constructor(
         return masterIndexId
     }
 
-    private fun addSource(
+    internal fun addSource(
         connection: Connection,
         type: SourceType,
-        cacheId: Int,
+        cacheId: Int?,
-        gameId: Int,
+        gameId: Int?,
         buildMajor: Int?,
         buildMinor: Int?,
         timestamp: Instant?,
@@ -552,7 +553,23 @@ public class CacheImporter @Inject constructor(
         description: String?,
         url: String?
     ): Int {
-        if (type == SourceType.JS5REMOTE && buildMajor != null) {
+        if (type == SourceType.CROSS_POLLINATION) {
+            connection.prepareStatement(
+                """
+                SELECT id
+                FROM sources
+                WHERE type = 'cross_pollination'
+            """.trimIndent()
+            ).use { stmt ->
+                stmt.executeQuery().use { rows ->
+                    if (rows.next()) {
+                        return rows.getInt(1)
+                    }
+                }
+            }
+        }
+
+        if (type == SourceType.JS5REMOTE && cacheId != null && gameId != null && buildMajor != null) {
             connection.prepareStatement(
                 """
                 SELECT id
@@ -581,8 +598,8 @@ public class CacheImporter @Inject constructor(
             """.trimIndent()
         ).use { stmt ->
             stmt.setString(1, type.toString().lowercase())
-            stmt.setInt(2, cacheId)
+            stmt.setObject(2, cacheId, Types.INTEGER)
-            stmt.setInt(3, gameId)
+            stmt.setObject(3, gameId, Types.INTEGER)
             stmt.setObject(4, buildMajor, Types.INTEGER)
             stmt.setObject(5, buildMinor, Types.INTEGER)
 
@@ -636,7 +653,7 @@ public class CacheImporter @Inject constructor(
         }
     }
 
-    private fun addGroups(connection: Connection, scopeId: Int, sourceId: Int, groups: List<Group>): List<Long> {
+    internal fun addGroups(connection: Connection, scopeId: Int, sourceId: Int, groups: List<Group>): List<Long> {
         val containerIds = addContainers(connection, groups)
 
         connection.prepareStatement(
@@ -795,7 +812,7 @@ public class CacheImporter @Inject constructor(
         return containerId
     }
 
-    private fun prepare(connection: Connection) {
+    internal fun prepare(connection: Connection) {
         connection.prepareStatement(
             """
             LOCK TABLE containers IN EXCLUSIVE MODE
@@ -1298,7 +1315,7 @@ public class CacheImporter @Inject constructor(
         }
     }
 
-    private fun addFiles(connection: Connection, sourceId: Int, files: List<File>) {
+    internal fun addFiles(connection: Connection, sourceId: Int, files: List<File>) {
         val blobIds = addBlobs(connection, files)
 
         connection.prepareStatement(

archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinateCommand.kt

@@ -0,0 +1,16 @@
+package org.openrs2.archive.cache
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.google.inject.Guice
+import kotlinx.coroutines.runBlocking
+import org.openrs2.archive.ArchiveModule
+import org.openrs2.inject.CloseableInjector
+
+public class CrossPollinateCommand : CliktCommand(name = "cross-pollinate") {
+    override fun run(): Unit = runBlocking {
+        CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector ->
+            val crossPollinator = injector.getInstance(CrossPollinator::class.java)
+            crossPollinator.crossPollinate()
+        }
+    }
+}

archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinator.kt

@@ -0,0 +1,223 @@
+package org.openrs2.archive.cache
+
+import io.netty.buffer.ByteBuf
+import io.netty.buffer.ByteBufAllocator
+import io.netty.buffer.ByteBufInputStream
+import io.netty.buffer.Unpooled
+import org.openrs2.buffer.crc32
+import org.openrs2.buffer.use
+import org.openrs2.cache.Js5Compression
+import org.openrs2.cache.Js5CompressionType
+import org.openrs2.db.Database
+import java.sql.Connection
+import java.util.zip.GZIPInputStream
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+public class CrossPollinator @Inject constructor(
+    private val database: Database,
+    private val alloc: ByteBufAllocator,
+    private val importer: CacheImporter
+) {
+    public suspend fun crossPollinate() {
+        database.execute { connection ->
+            for ((index, archive) in OLD_TO_NEW_ENGINE) {
+                crossPollinate(connection, index, archive);
+            }
+        }
+    }
+
+    private fun crossPollinate(connection: Connection, index: Int, archive: Int) {
+        val scopeId: Int
+
+        connection.prepareStatement(
+            """
+            SELECT id
+            FROM scopes
+            WHERE name = 'runescape'
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.executeQuery().use { rows ->
+                check(rows.next())
+
+                scopeId = rows.getInt(1)
+            }
+        }
+
+        val groups = mutableListOf<CacheImporter.Group>()
+        val files = mutableListOf<CacheImporter.File>()
+
+        try {
+            connection.prepareStatement(
+                """
+                SELECT
+                    new.group_id AS id,
+                    old.version AS old_version,
+                    old.crc32 AS old_crc32,
+                    b.data AS old_data,
+                    new.version AS new_version,
+                    new.crc32 AS new_crc32,
+                    c.data AS new_data
+                FROM (
+                    SELECT DISTINCT vf.index_id, vf.file_id, vf.version, vf.crc32
+                    FROM version_list_files vf
+                    WHERE vf.blob_id IN (
+                        SELECT v.blob_id
+                        FROM version_lists v
+                        JOIN resolved_archives a ON a.blob_id = v.blob_id AND a.archive_id = 5
+                    ) AND vf.index_id = ?
+                ) old
+                JOIN (
+                    SELECT DISTINCT ig.group_id, ig.version, ig.crc32
+                    FROM index_groups ig
+                    WHERE ig.container_id IN (
+                        SELECT i.container_id
+                        FROM resolved_indexes i
+                        WHERE i.scope_id = ? AND i.archive_id = ?
+                    )
+                ) new ON old.file_id = new.group_id AND old.version = new.version + 1
+                LEFT JOIN resolve_file(old.index_id, old.file_id, old.version, old.crc32) b ON TRUE
+                LEFT JOIN resolve_group(?, ?::uint1, new.group_id, new.crc32, new.version) c ON TRUE
+                WHERE (b.data IS NULL AND c.data IS NOT NULL) OR (b.data IS NOT NULL AND c.data IS NULL)
+            """.trimIndent()
+            ).use { stmt ->
+                stmt.setInt(1, index)
+                stmt.setInt(2, scopeId)
+                stmt.setInt(3, archive)
+                stmt.setInt(4, scopeId)
+                stmt.setInt(5, archive)
+
+                stmt.executeQuery().use { rows ->
+                    while (rows.next()) {
+                        val id = rows.getInt(1)
+                        val oldVersion = rows.getInt(2)
+                        val oldChecksum = rows.getInt(3)
+                        val newVersion = rows.getInt(5)
+                        val newChecksum = rows.getInt(6)
+
+                        val oldData = rows.getBytes(4)
+                        if (oldData != null) {
+                            Unpooled.wrappedBuffer(oldData).use { oldBuf ->
+                                fileToGroup(oldBuf, newChecksum).use { newBuf ->
+                                    if (newBuf != null) {
+                                        val uncompressed = Js5Compression.uncompressUnlessEncrypted(newBuf.slice())
+                                        groups += CacheImporter.Group(
+                                            archive,
+                                            id,
+                                            newBuf.retain(),
+                                            uncompressed,
+                                            newVersion,
+                                            false
+                                        )
+                                    }
+                                }
+                            }
+                        }
+
+                        val newData = rows.getBytes(7)
+                        if (newData != null) {
+                            Unpooled.wrappedBuffer(newData).use { newBuf ->
+                                val oldBuf = groupToFile(newBuf, oldChecksum)
+                                if (oldBuf != null) {
+                                    files += CacheImporter.File(index, id, oldBuf, oldVersion)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            if (groups.isEmpty() && files.isEmpty()) {
+                return
+            }
+
+            importer.prepare(connection)
+
+            val sourceId = importer.addSource(
+                connection,
+                type = CacheImporter.SourceType.CROSS_POLLINATION,
+                cacheId = null,
+                gameId = null,
+                buildMajor = null,
+                buildMinor = null,
+                timestamp = null,
+                name = null,
+                description = null,
+                url = null,
+            )
+
+            if (groups.isNotEmpty()) {
+                importer.addGroups(connection, scopeId, sourceId, groups)
+            }
+
+            if (files.isNotEmpty()) {
+                importer.addFiles(connection, sourceId, files)
+            }
+        } finally {
+            groups.forEach(CacheImporter.Group::release)
+            files.forEach(CacheImporter.File::release)
+        }
+    }
+
+    private fun getUncompressedLength(buf: ByteBuf): Int {
+        GZIPInputStream(ByteBufInputStream(buf)).use { input ->
+            var len = 0
+            val temp = ByteArray(4096)
+
+            while (true) {
+                val n = input.read(temp)
+                if (n == -1) {
+                    break
+                }
+                len += n
+            }
+
+            return len
+        }
+    }
+
+    private fun fileToGroup(input: ByteBuf, expectedChecksum: Int): ByteBuf? {
+        val len = input.readableBytes()
+        val lenWithHeader = len + JS5_COMPRESSION_HEADER_LEN
+        val uncompressedLen = getUncompressedLength(input.slice())
+
+        alloc.buffer(lenWithHeader, lenWithHeader).use { output ->
+            output.writeByte(Js5CompressionType.GZIP.ordinal)
+            output.writeInt(len)
+            output.writeInt(uncompressedLen)
+            output.writeBytes(input)
+
+            return if (output.crc32() == expectedChecksum) {
+                output.retain()
+            } else {
+                null
+            }
+        }
+    }
+
+    private fun groupToFile(input: ByteBuf, expectedChecksum: Int): ByteBuf? {
+        val type = Js5CompressionType.fromOrdinal(input.readUnsignedByte().toInt())
+        if (type != Js5CompressionType.GZIP) {
+            return null
+        }
+
+        input.skipBytes(JS5_COMPRESSION_HEADER_LEN - 1)
+
+        return if (input.crc32() == expectedChecksum) {
+            input.retainedSlice()
+        } else {
+            null
+        }
+    }
+
+    private companion object {
+        private val OLD_TO_NEW_ENGINE = mapOf(
+            1 to 7, // MODELS
+            3 to 6, // MIDI_SONGS
+            4 to 5, // MAPS
+        )
+
+        private const val JS5_COMPRESSION_HEADER_LEN = 9
+    }
+}

archive/src/main/resources/org/openrs2/archive/migrations/V19__source_type_cross_pollination.sql

@@ -0,0 +1,3 @@
+-- @formatter:off
+
+ALTER TYPE source_type ADD VALUE 'cross_pollination';

archive/src/main/resources/org/openrs2/archive/migrations/V20__cross_pollination.sql

@@ -0,0 +1,7 @@
+-- @formatter:off
+
+ALTER TABLE sources
+    ALTER COLUMN cache_id DROP NOT NULL,
+    ALTER COLUMN game_id DROP NOT NULL;
+
+CREATE UNIQUE INDEX ON sources (type) WHERE type = 'cross_pollination';