From 30b605d719d24df2854443ffc2096b5b0a705b35 Mon Sep 17 00:00:00 2001 From: Graham Date: Tue, 22 Dec 2020 21:18:25 +0000 Subject: [PATCH] Store master indexes in the database I have a small collection of these from when I ran a service that polled the JS5 server for the master index back in 2009. We'll probably be able to find some in old Rune-Server threads too. They'll be useful for verifying caches with an unclear provenance. Signed-off-by: Graham --- .../org/openrs2/archive/ArchiveCommand.kt | 2 + .../openrs2/archive/cache/CacheImporter.kt | 105 +----------------- .../openrs2/archive/container/Container.kt | 16 +++ .../archive/container/ContainerImporter.kt | 94 ++++++++++++++++ .../archive/masterindex/ImportCommand.kt | 28 +++++ .../archive/masterindex/MasterIndexCommand.kt | 12 ++ .../masterindex/MasterIndexImporter.kt | 63 +++++++++++ .../org/openrs2/archive/V1__init.sql | 13 +++ 8 files changed, 233 insertions(+), 100 deletions(-) create mode 100644 archive/src/main/kotlin/org/openrs2/archive/container/Container.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/container/ContainerImporter.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/masterindex/ImportCommand.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexCommand.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexImporter.kt diff --git a/archive/src/main/kotlin/org/openrs2/archive/ArchiveCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/ArchiveCommand.kt index 95ce488d24..980ea5bf5d 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/ArchiveCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/ArchiveCommand.kt @@ -4,6 +4,7 @@ import com.github.ajalt.clikt.core.NoOpCliktCommand import com.github.ajalt.clikt.core.subcommands import org.openrs2.archive.cache.CacheCommand import org.openrs2.archive.key.KeyCommand +import org.openrs2.archive.masterindex.MasterIndexCommand import org.openrs2.archive.name.NameCommand public fun main(args: Array): Unit = ArchiveCommand().main(args) @@ -13,6 +14,7 @@ public class ArchiveCommand : NoOpCliktCommand(name = "archive") { subcommands( CacheCommand(), KeyCommand(), + MasterIndexCommand(), NameCommand() ) } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index 8ce22938ee..8fd387292a 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -3,8 +3,8 @@ package org.openrs2.archive.cache import io.netty.buffer.ByteBuf import io.netty.buffer.ByteBufAllocator import io.netty.buffer.ByteBufUtil -import io.netty.buffer.DefaultByteBufHolder -import org.openrs2.buffer.crc32 +import org.openrs2.archive.container.Container +import org.openrs2.archive.container.ContainerImporter import org.openrs2.buffer.use import org.openrs2.cache.Js5Archive import org.openrs2.cache.Js5Compression @@ -24,15 +24,6 @@ public class CacheImporter @Inject constructor( private val database: Database, private val alloc: ByteBufAllocator ) { - private abstract class Container( - data: ByteBuf - ) : DefaultByteBufHolder(data) { - val bytes: ByteArray = ByteBufUtil.getBytes(data, data.readerIndex(), data.readableBytes(), false) - val crc32 = data.crc32() - val whirlpool = Whirlpool.whirlpool(bytes) - abstract val encrypted: Boolean - } - private class Index( val archive: Int, val index: Js5Index, @@ -51,28 +42,7 @@ public class CacheImporter @Inject constructor( public suspend fun import(store: Store) { database.execute { connection -> - connection.prepareStatement( - """ - LOCK TABLE containers IN EXCLUSIVE MODE - """.trimIndent() - ).use { stmt -> - stmt.execute() - } - - // create temporary tables - connection.prepareStatement( - """ - CREATE TEMPORARY TABLE tmp_containers ( - index INTEGER NOT NULL, - crc32 INTEGER NOT NULL, - whirlpool BYTEA NOT NULL, - data BYTEA NOT NULL, - encrypted BOOLEAN NOT NULL - ) ON COMMIT DROP - """.trimIndent() - ).use { stmt -> - stmt.execute() - } + ContainerImporter.prepare(connection) // import indexes val indexes = mutableListOf() @@ -132,73 +102,8 @@ public class CacheImporter @Inject constructor( } } - private fun addContainer(connection: Connection, container: Container): Long { - return addContainers(connection, listOf(container)).single() - } - - private fun addContainers(connection: Connection, containers: List): List { - connection.prepareStatement( - """ - TRUNCATE TABLE tmp_containers - """.trimIndent() - ).use { stmt -> - stmt.execute() - } - - connection.prepareStatement( - """ - INSERT INTO tmp_containers (index, crc32, whirlpool, data, encrypted) - VALUES (?, ?, ?, ?, ?) - """.trimIndent() - ).use { stmt -> - for ((i, container) in containers.withIndex()) { - stmt.setInt(1, i) - stmt.setInt(2, container.crc32) - stmt.setBytes(3, container.whirlpool) - stmt.setBytes(4, container.bytes) - stmt.setBoolean(5, container.encrypted) - stmt.addBatch() - } - - stmt.executeBatch() - } - - connection.prepareStatement( - """ - INSERT INTO containers (crc32, whirlpool, data, encrypted) - SELECT t.crc32, t.whirlpool, t.data, t.encrypted - FROM tmp_containers t - LEFT JOIN containers c ON c.whirlpool = t.whirlpool - WHERE c.whirlpool IS NULL - ON CONFLICT DO NOTHING - """.trimIndent() - ).use { stmt -> - stmt.execute() - } - - val ids = mutableListOf() - - connection.prepareStatement( - """ - SELECT c.id - FROM tmp_containers t - JOIN containers c ON c.whirlpool = t.whirlpool - ORDER BY t.index ASC - """.trimIndent() - ).use { stmt -> - stmt.executeQuery().use { rows -> - while (rows.next()) { - ids += rows.getLong(1) - } - } - } - - check(ids.size == containers.size) - return ids - } - private fun addGroups(connection: Connection, groups: List) { - val containerIds = addContainers(connection, groups) + val containerIds = ContainerImporter.addContainers(connection, groups) connection.prepareStatement( """ @@ -289,7 +194,7 @@ public class CacheImporter @Inject constructor( // TODO(gpe): skip most of this function if we encounter a conflict? private fun addIndex(connection: Connection, cacheId: Long, index: Index) { - val containerId = addContainer(connection, index) + val containerId = ContainerImporter.addContainer(connection, index) connection.prepareStatement( """ diff --git a/archive/src/main/kotlin/org/openrs2/archive/container/Container.kt b/archive/src/main/kotlin/org/openrs2/archive/container/Container.kt new file mode 100644 index 0000000000..fdcfca41e4 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/container/Container.kt @@ -0,0 +1,16 @@ +package org.openrs2.archive.container + +import io.netty.buffer.ByteBuf +import io.netty.buffer.ByteBufUtil +import io.netty.buffer.DefaultByteBufHolder +import org.openrs2.buffer.crc32 +import org.openrs2.crypto.Whirlpool + +public abstract class Container( + data: ByteBuf +) : DefaultByteBufHolder(data) { + public val bytes: ByteArray = ByteBufUtil.getBytes(data, data.readerIndex(), data.readableBytes(), false) + public val crc32: Int = data.crc32() + public val whirlpool: ByteArray = Whirlpool.whirlpool(bytes) + public abstract val encrypted: Boolean +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/container/ContainerImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/container/ContainerImporter.kt new file mode 100644 index 0000000000..d1874108a9 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/container/ContainerImporter.kt @@ -0,0 +1,94 @@ +package org.openrs2.archive.container + +import java.sql.Connection + +public object ContainerImporter { + public fun prepare(connection: Connection) { + connection.prepareStatement( + """ + LOCK TABLE containers IN EXCLUSIVE MODE + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + CREATE TEMPORARY TABLE tmp_containers ( + index INTEGER NOT NULL, + crc32 INTEGER NOT NULL, + whirlpool BYTEA NOT NULL, + data BYTEA NOT NULL, + encrypted BOOLEAN NOT NULL + ) ON COMMIT DROP + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + } + + public fun addContainer(connection: Connection, container: Container): Long { + return addContainers(connection, listOf(container)).single() + } + + public fun addContainers(connection: Connection, containers: List): List { + connection.prepareStatement( + """ + TRUNCATE TABLE tmp_containers + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO tmp_containers (index, crc32, whirlpool, data, encrypted) + VALUES (?, ?, ?, ?, ?) + """.trimIndent() + ).use { stmt -> + for ((i, container) in containers.withIndex()) { + stmt.setInt(1, i) + stmt.setInt(2, container.crc32) + stmt.setBytes(3, container.whirlpool) + stmt.setBytes(4, container.bytes) + stmt.setBoolean(5, container.encrypted) + stmt.addBatch() + } + + stmt.executeBatch() + } + + connection.prepareStatement( + """ + INSERT INTO containers (crc32, whirlpool, data, encrypted) + SELECT t.crc32, t.whirlpool, t.data, t.encrypted + FROM tmp_containers t + LEFT JOIN containers c ON c.whirlpool = t.whirlpool + WHERE c.whirlpool IS NULL + ON CONFLICT DO NOTHING + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + val ids = mutableListOf() + + connection.prepareStatement( + """ + SELECT c.id + FROM tmp_containers t + JOIN containers c ON c.whirlpool = t.whirlpool + ORDER BY t.index ASC + """.trimIndent() + ).use { stmt -> + stmt.executeQuery().use { rows -> + while (rows.next()) { + ids += rows.getLong(1) + } + } + } + + check(ids.size == containers.size) + return ids + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/masterindex/ImportCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/masterindex/ImportCommand.kt new file mode 100644 index 0000000000..6d689e2846 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/masterindex/ImportCommand.kt @@ -0,0 +1,28 @@ +package org.openrs2.archive.masterindex + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.types.path +import com.google.inject.Guice +import io.netty.buffer.Unpooled +import kotlinx.coroutines.runBlocking +import org.openrs2.archive.ArchiveModule +import org.openrs2.buffer.use +import java.nio.file.Files + +public class ImportCommand : CliktCommand(name = "import") { + private val input by argument().path( + mustExist = true, + canBeDir = false, + mustBeReadable = true + ) + + override fun run(): Unit = runBlocking { + val injector = Guice.createInjector(ArchiveModule) + val importer = injector.getInstance(MasterIndexImporter::class.java) + + Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> + importer.import(buf) + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexCommand.kt new file mode 100644 index 0000000000..32fb569a02 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexCommand.kt @@ -0,0 +1,12 @@ +package org.openrs2.archive.masterindex + +import com.github.ajalt.clikt.core.NoOpCliktCommand +import com.github.ajalt.clikt.core.subcommands + +public class MasterIndexCommand : NoOpCliktCommand(name = "master-index") { + init { + subcommands( + ImportCommand() + ) + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexImporter.kt new file mode 100644 index 0000000000..cac49e22ba --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/masterindex/MasterIndexImporter.kt @@ -0,0 +1,63 @@ +package org.openrs2.archive.masterindex + +import io.netty.buffer.ByteBuf +import org.openrs2.archive.container.Container +import org.openrs2.archive.container.ContainerImporter +import org.openrs2.buffer.use +import org.openrs2.cache.Js5Compression +import org.openrs2.cache.Js5MasterIndex +import org.openrs2.db.Database +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +public class MasterIndexImporter @Inject constructor( + private val database: Database +) { + private class MasterIndex( + data: ByteBuf, + val index: Js5MasterIndex + ) : Container(data) { + override val encrypted: Boolean = false + } + + public suspend fun import(buf: ByteBuf) { + database.execute { connection -> + ContainerImporter.prepare(connection) + + val masterIndex = Js5Compression.uncompress(buf).use { uncompressed -> + MasterIndex(buf, Js5MasterIndex.read(uncompressed)) + } + + val containerId = ContainerImporter.addContainer(connection, masterIndex) + + connection.prepareStatement( + """ + INSERT INTO master_indexes (container_id) + VALUES (?) + """.trimIndent() + ).use { stmt -> + stmt.setLong(1, containerId) + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO master_index_entries (container_id, archive_id, crc32, version) + VALUES (?, ?, ?, ?) + """.trimIndent() + ).use { stmt -> + for ((i, entry) in masterIndex.index.entries.withIndex()) { + stmt.setLong(1, containerId) + stmt.setInt(2, i) + stmt.setInt(3, entry.checksum) + stmt.setInt(4, entry.version) + + stmt.addBatch() + } + + stmt.executeBatch() + } + } + } +} diff --git a/archive/src/main/resources/org/openrs2/archive/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/V1__init.sql index 8071b03812..e8822779eb 100644 --- a/archive/src/main/resources/org/openrs2/archive/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/V1__init.sql @@ -66,6 +66,19 @@ CREATE TABLE index_files ( FOREIGN KEY (container_id, group_id) REFERENCES index_groups (container_id, group_id) ); +CREATE TABLE master_indexes ( + container_id BIGINT PRIMARY KEY NOT NULL REFERENCES containers (id) +); + +CREATE TABLE master_index_entries ( + container_id BIGINT NOT NULL REFERENCES master_indexes (container_id), + archive_id uint1 NOT NULL, + crc32 INTEGER NOT NULL, + whirlpool BYTEA NULL, + version INTEGER NOT NULL, + PRIMARY KEY (container_id, archive_id) +); + CREATE TABLE caches ( id BIGSERIAL PRIMARY KEY NOT NULL, -- This doesn't correspond to a hash used by the client - it was just