From 28a9667471aef1d9e6df4c9af531b2664931a523 Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 24 Jun 2023 16:11:00 +0100 Subject: [PATCH] Add support for resolving files/groups between old and new engine caches Signed-off-by: Graham --- .../org/openrs2/archive/cache/CacheCommand.kt | 1 + .../openrs2/archive/cache/CacheImporter.kt | 39 ++- .../archive/cache/CrossPollinateCommand.kt | 16 ++ .../openrs2/archive/cache/CrossPollinator.kt | 223 ++++++++++++++++++ .../V19__source_type_cross_pollination.sql | 3 + .../migrations/V20__cross_pollination.sql | 7 + 6 files changed, 278 insertions(+), 11 deletions(-) create mode 100644 archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinateCommand.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinator.kt create mode 100644 archive/src/main/resources/org/openrs2/archive/migrations/V19__source_type_cross_pollination.sql create mode 100644 archive/src/main/resources/org/openrs2/archive/migrations/V20__cross_pollination.sql diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheCommand.kt index 196b9e49..e0776103 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheCommand.kt @@ -7,6 +7,7 @@ import org.openrs2.archive.cache.finder.ExtractCommand public class CacheCommand : NoOpCliktCommand(name = "cache") { init { subcommands( + CrossPollinateCommand(), DownloadCommand(), ExtractCommand(), ImportCommand(), diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index df52a17e..74b4ab3a 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -105,9 +105,10 @@ public class CacheImporter @Inject constructor( public val version: Int ) : Blob(buf) - private enum class SourceType { + internal enum class SourceType { DISK, - JS5REMOTE + JS5REMOTE, + CROSS_POLLINATION } public data class MasterIndexResult( @@ -540,11 +541,11 @@ public class CacheImporter @Inject constructor( return masterIndexId } - private fun addSource( + internal fun addSource( connection: Connection, type: SourceType, - cacheId: Int, - gameId: Int, + cacheId: Int?, + gameId: Int?, buildMajor: Int?, buildMinor: Int?, timestamp: Instant?, @@ -552,7 +553,23 @@ public class CacheImporter @Inject constructor( description: String?, url: String? ): Int { - if (type == SourceType.JS5REMOTE && buildMajor != null) { + if (type == SourceType.CROSS_POLLINATION) { + connection.prepareStatement( + """ + SELECT id + FROM sources + WHERE type = 'cross_pollination' + """.trimIndent() + ).use { stmt -> + stmt.executeQuery().use { rows -> + if (rows.next()) { + return rows.getInt(1) + } + } + } + } + + if (type == SourceType.JS5REMOTE && cacheId != null && gameId != null && buildMajor != null) { connection.prepareStatement( """ SELECT id @@ -581,8 +598,8 @@ public class CacheImporter @Inject constructor( """.trimIndent() ).use { stmt -> stmt.setString(1, type.toString().lowercase()) - stmt.setInt(2, cacheId) - stmt.setInt(3, gameId) + stmt.setObject(2, cacheId, Types.INTEGER) + stmt.setObject(3, gameId, Types.INTEGER) stmt.setObject(4, buildMajor, Types.INTEGER) stmt.setObject(5, buildMinor, Types.INTEGER) @@ -636,7 +653,7 @@ public class CacheImporter @Inject constructor( } } - private fun addGroups(connection: Connection, scopeId: Int, sourceId: Int, groups: List): List { + internal fun addGroups(connection: Connection, scopeId: Int, sourceId: Int, groups: List): List { val containerIds = addContainers(connection, groups) connection.prepareStatement( @@ -795,7 +812,7 @@ public class CacheImporter @Inject constructor( return containerId } - private fun prepare(connection: Connection) { + internal fun prepare(connection: Connection) { connection.prepareStatement( """ LOCK TABLE containers IN EXCLUSIVE MODE @@ -1298,7 +1315,7 @@ public class CacheImporter @Inject constructor( } } - private fun addFiles(connection: Connection, sourceId: Int, files: List) { + internal fun addFiles(connection: Connection, sourceId: Int, files: List) { val blobIds = addBlobs(connection, files) connection.prepareStatement( diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinateCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinateCommand.kt new file mode 100644 index 00000000..f9bdd2f1 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinateCommand.kt @@ -0,0 +1,16 @@ +package org.openrs2.archive.cache + +import com.github.ajalt.clikt.core.CliktCommand +import com.google.inject.Guice +import kotlinx.coroutines.runBlocking +import org.openrs2.archive.ArchiveModule +import org.openrs2.inject.CloseableInjector + +public class CrossPollinateCommand : CliktCommand(name = "cross-pollinate") { + override fun run(): Unit = runBlocking { + CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> + val crossPollinator = injector.getInstance(CrossPollinator::class.java) + crossPollinator.crossPollinate() + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinator.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinator.kt new file mode 100644 index 00000000..91a7c194 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CrossPollinator.kt @@ -0,0 +1,223 @@ +package org.openrs2.archive.cache + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufAllocator +import io.netty.buffer.ByteBufInputStream +import io.netty.buffer.Unpooled +import org.openrs2.buffer.crc32 +import org.openrs2.buffer.use +import org.openrs2.cache.Js5Compression +import org.openrs2.cache.Js5CompressionType +import org.openrs2.db.Database +import java.sql.Connection +import java.util.zip.GZIPInputStream +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +public class CrossPollinator @Inject constructor( + private val database: Database, + private val alloc: ByteBufAllocator, + private val importer: CacheImporter +) { + public suspend fun crossPollinate() { + database.execute { connection -> + for ((index, archive) in OLD_TO_NEW_ENGINE) { + crossPollinate(connection, index, archive); + } + } + } + + private fun crossPollinate(connection: Connection, index: Int, archive: Int) { + val scopeId: Int + + connection.prepareStatement( + """ + SELECT id + FROM scopes + WHERE name = 'runescape' + """.trimIndent() + ).use { stmt -> + stmt.executeQuery().use { rows -> + check(rows.next()) + + scopeId = rows.getInt(1) + } + } + + val groups = mutableListOf() + val files = mutableListOf() + + try { + connection.prepareStatement( + """ + SELECT + new.group_id AS id, + old.version AS old_version, + old.crc32 AS old_crc32, + b.data AS old_data, + new.version AS new_version, + new.crc32 AS new_crc32, + c.data AS new_data + FROM ( + SELECT DISTINCT vf.index_id, vf.file_id, vf.version, vf.crc32 + FROM version_list_files vf + WHERE vf.blob_id IN ( + SELECT v.blob_id + FROM version_lists v + JOIN resolved_archives a ON a.blob_id = v.blob_id AND a.archive_id = 5 + ) AND vf.index_id = ? + ) old + JOIN ( + SELECT DISTINCT ig.group_id, ig.version, ig.crc32 + FROM index_groups ig + WHERE ig.container_id IN ( + SELECT i.container_id + FROM resolved_indexes i + WHERE i.scope_id = ? AND i.archive_id = ? + ) + ) new ON old.file_id = new.group_id AND old.version = new.version + 1 + LEFT JOIN resolve_file(old.index_id, old.file_id, old.version, old.crc32) b ON TRUE + LEFT JOIN resolve_group(?, ?::uint1, new.group_id, new.crc32, new.version) c ON TRUE + WHERE (b.data IS NULL AND c.data IS NOT NULL) OR (b.data IS NOT NULL AND c.data IS NULL) + """.trimIndent() + ).use { stmt -> + stmt.setInt(1, index) + stmt.setInt(2, scopeId) + stmt.setInt(3, archive) + stmt.setInt(4, scopeId) + stmt.setInt(5, archive) + + stmt.executeQuery().use { rows -> + while (rows.next()) { + val id = rows.getInt(1) + val oldVersion = rows.getInt(2) + val oldChecksum = rows.getInt(3) + val newVersion = rows.getInt(5) + val newChecksum = rows.getInt(6) + + val oldData = rows.getBytes(4) + if (oldData != null) { + Unpooled.wrappedBuffer(oldData).use { oldBuf -> + fileToGroup(oldBuf, newChecksum).use { newBuf -> + if (newBuf != null) { + val uncompressed = Js5Compression.uncompressUnlessEncrypted(newBuf.slice()) + groups += CacheImporter.Group( + archive, + id, + newBuf.retain(), + uncompressed, + newVersion, + false + ) + } + } + } + } + + val newData = rows.getBytes(7) + if (newData != null) { + Unpooled.wrappedBuffer(newData).use { newBuf -> + val oldBuf = groupToFile(newBuf, oldChecksum) + if (oldBuf != null) { + files += CacheImporter.File(index, id, oldBuf, oldVersion) + } + } + } + } + } + } + + if (groups.isEmpty() && files.isEmpty()) { + return + } + + importer.prepare(connection) + + val sourceId = importer.addSource( + connection, + type = CacheImporter.SourceType.CROSS_POLLINATION, + cacheId = null, + gameId = null, + buildMajor = null, + buildMinor = null, + timestamp = null, + name = null, + description = null, + url = null, + ) + + if (groups.isNotEmpty()) { + importer.addGroups(connection, scopeId, sourceId, groups) + } + + if (files.isNotEmpty()) { + importer.addFiles(connection, sourceId, files) + } + } finally { + groups.forEach(CacheImporter.Group::release) + files.forEach(CacheImporter.File::release) + } + } + + private fun getUncompressedLength(buf: ByteBuf): Int { + GZIPInputStream(ByteBufInputStream(buf)).use { input -> + var len = 0 + val temp = ByteArray(4096) + + while (true) { + val n = input.read(temp) + if (n == -1) { + break + } + len += n + } + + return len + } + } + + private fun fileToGroup(input: ByteBuf, expectedChecksum: Int): ByteBuf? { + val len = input.readableBytes() + val lenWithHeader = len + JS5_COMPRESSION_HEADER_LEN + val uncompressedLen = getUncompressedLength(input.slice()) + + alloc.buffer(lenWithHeader, lenWithHeader).use { output -> + output.writeByte(Js5CompressionType.GZIP.ordinal) + output.writeInt(len) + output.writeInt(uncompressedLen) + output.writeBytes(input) + + return if (output.crc32() == expectedChecksum) { + output.retain() + } else { + null + } + } + } + + private fun groupToFile(input: ByteBuf, expectedChecksum: Int): ByteBuf? { + val type = Js5CompressionType.fromOrdinal(input.readUnsignedByte().toInt()) + if (type != Js5CompressionType.GZIP) { + return null + } + + input.skipBytes(JS5_COMPRESSION_HEADER_LEN - 1) + + return if (input.crc32() == expectedChecksum) { + input.retainedSlice() + } else { + null + } + } + + private companion object { + private val OLD_TO_NEW_ENGINE = mapOf( + 1 to 7, // MODELS + 3 to 6, // MIDI_SONGS + 4 to 5, // MAPS + ) + + private const val JS5_COMPRESSION_HEADER_LEN = 9 + } +} diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V19__source_type_cross_pollination.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V19__source_type_cross_pollination.sql new file mode 100644 index 00000000..4c40b5f7 --- /dev/null +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V19__source_type_cross_pollination.sql @@ -0,0 +1,3 @@ +-- @formatter:off + +ALTER TYPE source_type ADD VALUE 'cross_pollination'; diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V20__cross_pollination.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V20__cross_pollination.sql new file mode 100644 index 00000000..5a6384ee --- /dev/null +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V20__cross_pollination.sql @@ -0,0 +1,7 @@ +-- @formatter:off + +ALTER TABLE sources + ALTER COLUMN cache_id DROP NOT NULL, + ALTER COLUMN game_id DROP NOT NULL; + +CREATE UNIQUE INDEX ON sources (type) WHERE type = 'cross_pollination';