From 77e227c93dec1ca413212b33bc3dced7e826556e Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 14 Jan 2021 18:35:56 +0000 Subject: [PATCH] Store game, build and timestamp in the master index table Signed-off-by: Graham --- archive/build.gradle.kts | 1 + .../openrs2/archive/cache/CacheImporter.kt | 62 ++++++++++++++++--- .../openrs2/archive/cache/ImportCommand.kt | 9 ++- .../archive/cache/ImportMasterIndexCommand.kt | 9 ++- .../archive/cache/Js5ChannelHandler.kt | 5 +- .../org/openrs2/archive/V1__init.sql | 5 +- 6 files changed, 78 insertions(+), 13 deletions(-) diff --git a/archive/build.gradle.kts b/archive/build.gradle.kts index 7718eb2cc8..66dbecf186 100644 --- a/archive/build.gradle.kts +++ b/archive/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { implementation(project(":buffer")) implementation(project(":cache")) + implementation(project(":cli")) implementation(project(":db")) implementation(project(":json")) implementation(project(":net")) diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index d119f49cb5..cea95b49f3 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -21,6 +21,9 @@ import java.io.IOException import java.sql.Connection import java.sql.SQLException import java.sql.Types +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset import javax.inject.Inject import javax.inject.Singleton @@ -60,14 +63,16 @@ public class CacheImporter @Inject constructor( override val encrypted: Boolean ) : Container(data) - public suspend fun import(store: Store) { + public suspend fun import(store: Store, game: String, build: Int?, timestamp: Instant?) { database.execute { connection -> prepare(connection) + val gameId = getGameId(connection, game) + // import master index val masterIndex = createMasterIndex(store) try { - addMasterIndex(connection, masterIndex) + addMasterIndex(connection, masterIndex, gameId, build, timestamp) } finally { masterIndex.release() } @@ -116,13 +121,15 @@ public class CacheImporter @Inject constructor( } } - public suspend fun importMasterIndex(buf: ByteBuf) { + public suspend fun importMasterIndex(buf: ByteBuf, game: String, build: Int?, timestamp: Instant?) { Js5Compression.uncompress(buf.slice()).use { uncompressed -> val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf) database.execute { connection -> prepare(connection) - addMasterIndex(connection, masterIndex) + + val gameId = getGameId(connection, game) + addMasterIndex(connection, masterIndex, gameId, build, timestamp) } } } @@ -131,7 +138,8 @@ public class CacheImporter @Inject constructor( masterIndex: Js5MasterIndex, buf: ByteBuf, gameId: Int, - build: Int + build: Int, + timestamp: Instant ): List { return database.execute { connection -> prepare(connection) @@ -149,7 +157,7 @@ public class CacheImporter @Inject constructor( stmt.execute() } - addMasterIndex(connection, MasterIndex(masterIndex, buf)) + addMasterIndex(connection, MasterIndex(masterIndex, buf), gameId, build, timestamp) connection.prepareStatement( """ @@ -293,17 +301,33 @@ public class CacheImporter @Inject constructor( } } - private fun addMasterIndex(connection: Connection, masterIndex: MasterIndex) { + private fun addMasterIndex( + connection: Connection, + masterIndex: MasterIndex, + gameId: Int, + build: Int? = null, + timestamp: Instant? = null + ) { val containerId = addContainer(connection, masterIndex) val savepoint = connection.setSavepoint() + // TODO(gpe): override game_id/build/timestamp if they're null? connection.prepareStatement( """ - INSERT INTO master_indexes (container_id) - VALUES (?) + INSERT INTO master_indexes (container_id, game_id, build, timestamp) + VALUES (?, ?, ?, ?) """.trimIndent() ).use { stmt -> stmt.setLong(1, containerId) + stmt.setObject(2, gameId, Types.INTEGER) + stmt.setObject(3, build, Types.INTEGER) + + if (timestamp != null) { + val offsetDateTime = OffsetDateTime.ofInstant(timestamp, ZoneOffset.UTC) + stmt.setObject(4, offsetDateTime, Types.TIMESTAMP_WITH_TIMEZONE) + } else { + stmt.setNull(4, Types.TIMESTAMP_WITH_TIMEZONE) + } try { stmt.execute() @@ -541,6 +565,26 @@ public class CacheImporter @Inject constructor( return ids } + private fun getGameId(connection: Connection, name: String): Int { + connection.prepareStatement( + """ + SELECT id + FROM games + WHERE name = ? + """.trimIndent() + ).use { stmt -> + stmt.setString(1, name) + + stmt.executeQuery().use { rows -> + if (!rows.next()) { + throw Exception("Game not found") + } + + return rows.getInt(1) + } + } + } + public companion object { public const val BATCH_SIZE: Int = 1024 } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt index ecdb369e8d..4cfc4e75dc 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt @@ -2,13 +2,20 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.path import com.google.inject.Guice import kotlinx.coroutines.runBlocking import org.openrs2.archive.ArchiveModule import org.openrs2.cache.Store +import org.openrs2.cli.instant public class ImportCommand : CliktCommand(name = "import") { + private val build by option().int() + private val timestamp by option().instant() + + private val game by argument() private val input by argument().path( mustExist = true, canBeFile = false, @@ -20,7 +27,7 @@ public class ImportCommand : CliktCommand(name = "import") { val importer = injector.getInstance(CacheImporter::class.java) Store.open(input).use { store -> - importer.import(store) + importer.import(store, game, build, timestamp) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt index 6ec8b39343..f576cfc12e 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt @@ -2,15 +2,22 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.path import com.google.inject.Guice import io.netty.buffer.Unpooled import kotlinx.coroutines.runBlocking import org.openrs2.archive.ArchiveModule import org.openrs2.buffer.use +import org.openrs2.cli.instant import java.nio.file.Files public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index") { + private val build by option().int() + private val timestamp by option().instant() + + private val game by argument() private val input by argument().path( mustExist = true, canBeDir = false, @@ -22,7 +29,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index val importer = injector.getInstance(CacheImporter::class.java) Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> - importer.importMasterIndex(buf) + importer.importMasterIndex(buf, game, build, timestamp) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt index 5b46faf1c3..c3c2d4fde4 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt @@ -22,6 +22,7 @@ import org.openrs2.protocol.js5.Js5ResponseDecoder import org.openrs2.protocol.js5.XorDecoder import org.openrs2.protocol.login.LoginRequest import org.openrs2.protocol.login.LoginResponse +import java.time.Instant import kotlin.coroutines.Continuation import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException @@ -156,7 +157,9 @@ public class Js5ChannelHandler( Js5MasterIndex.read(uncompressed) } - val rawIndexes = runBlocking { importer.importMasterIndexAndGetIndexes(masterIndex!!, buf, gameId, build) } + val rawIndexes = runBlocking { + importer.importMasterIndexAndGetIndexes(masterIndex!!, buf, gameId, build, Instant.now()) + } try { indexes = arrayOfNulls(rawIndexes.size) diff --git a/archive/src/main/resources/org/openrs2/archive/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/V1__init.sql index a7f928a6ac..d9881c7fbc 100644 --- a/archive/src/main/resources/org/openrs2/archive/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/V1__init.sql @@ -80,7 +80,10 @@ CREATE TABLE index_files ( ); CREATE TABLE master_indexes ( - container_id BIGINT PRIMARY KEY NOT NULL REFERENCES containers (id) + container_id BIGINT PRIMARY KEY NOT NULL REFERENCES containers (id), + game_id INTEGER NOT NULL REFERENCES games (id), + build INTEGER NULL, + timestamp TIMESTAMPTZ NULL ); CREATE TABLE master_index_entries (