Store game, build and timestamp in the master index table

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent ab006bbf44
commit 77e227c93d
  1. 1
      archive/build.gradle.kts
  2. 62
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  3. 9
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt
  4. 9
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt
  5. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt
  6. 5
      archive/src/main/resources/org/openrs2/archive/V1__init.sql

@ -13,6 +13,7 @@ dependencies {
implementation(project(":buffer")) implementation(project(":buffer"))
implementation(project(":cache")) implementation(project(":cache"))
implementation(project(":cli"))
implementation(project(":db")) implementation(project(":db"))
implementation(project(":json")) implementation(project(":json"))
implementation(project(":net")) implementation(project(":net"))

@ -21,6 +21,9 @@ import java.io.IOException
import java.sql.Connection import java.sql.Connection
import java.sql.SQLException import java.sql.SQLException
import java.sql.Types import java.sql.Types
import java.time.Instant
import java.time.OffsetDateTime
import java.time.ZoneOffset
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -60,14 +63,16 @@ public class CacheImporter @Inject constructor(
override val encrypted: Boolean override val encrypted: Boolean
) : Container(data) ) : Container(data)
public suspend fun import(store: Store) { public suspend fun import(store: Store, game: String, build: Int?, timestamp: Instant?) {
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
val gameId = getGameId(connection, game)
// import master index // import master index
val masterIndex = createMasterIndex(store) val masterIndex = createMasterIndex(store)
try { try {
addMasterIndex(connection, masterIndex) addMasterIndex(connection, masterIndex, gameId, build, timestamp)
} finally { } finally {
masterIndex.release() masterIndex.release()
} }
@ -116,13 +121,15 @@ public class CacheImporter @Inject constructor(
} }
} }
public suspend fun importMasterIndex(buf: ByteBuf) { public suspend fun importMasterIndex(buf: ByteBuf, game: String, build: Int?, timestamp: Instant?) {
Js5Compression.uncompress(buf.slice()).use { uncompressed -> Js5Compression.uncompress(buf.slice()).use { uncompressed ->
val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf) val masterIndex = MasterIndex(Js5MasterIndex.read(uncompressed.slice()), buf)
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
addMasterIndex(connection, masterIndex)
val gameId = getGameId(connection, game)
addMasterIndex(connection, masterIndex, gameId, build, timestamp)
} }
} }
} }
@ -131,7 +138,8 @@ public class CacheImporter @Inject constructor(
masterIndex: Js5MasterIndex, masterIndex: Js5MasterIndex,
buf: ByteBuf, buf: ByteBuf,
gameId: Int, gameId: Int,
build: Int build: Int,
timestamp: Instant
): List<ByteBuf?> { ): List<ByteBuf?> {
return database.execute { connection -> return database.execute { connection ->
prepare(connection) prepare(connection)
@ -149,7 +157,7 @@ public class CacheImporter @Inject constructor(
stmt.execute() stmt.execute()
} }
addMasterIndex(connection, MasterIndex(masterIndex, buf)) addMasterIndex(connection, MasterIndex(masterIndex, buf), gameId, build, timestamp)
connection.prepareStatement( connection.prepareStatement(
""" """
@ -293,17 +301,33 @@ public class CacheImporter @Inject constructor(
} }
} }
private fun addMasterIndex(connection: Connection, masterIndex: MasterIndex) { private fun addMasterIndex(
connection: Connection,
masterIndex: MasterIndex,
gameId: Int,
build: Int? = null,
timestamp: Instant? = null
) {
val containerId = addContainer(connection, masterIndex) val containerId = addContainer(connection, masterIndex)
val savepoint = connection.setSavepoint() val savepoint = connection.setSavepoint()
// TODO(gpe): override game_id/build/timestamp if they're null?
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO master_indexes (container_id) INSERT INTO master_indexes (container_id, game_id, build, timestamp)
VALUES (?) VALUES (?, ?, ?, ?)
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setLong(1, containerId) stmt.setLong(1, containerId)
stmt.setObject(2, gameId, Types.INTEGER)
stmt.setObject(3, build, Types.INTEGER)
if (timestamp != null) {
val offsetDateTime = OffsetDateTime.ofInstant(timestamp, ZoneOffset.UTC)
stmt.setObject(4, offsetDateTime, Types.TIMESTAMP_WITH_TIMEZONE)
} else {
stmt.setNull(4, Types.TIMESTAMP_WITH_TIMEZONE)
}
try { try {
stmt.execute() stmt.execute()
@ -541,6 +565,26 @@ public class CacheImporter @Inject constructor(
return ids return ids
} }
private fun getGameId(connection: Connection, name: String): Int {
connection.prepareStatement(
"""
SELECT id
FROM games
WHERE name = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, name)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
throw Exception("Game not found")
}
return rows.getInt(1)
}
}
}
public companion object { public companion object {
public const val BATCH_SIZE: Int = 1024 public const val BATCH_SIZE: Int = 1024
} }

@ -2,13 +2,20 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice import com.google.inject.Guice
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule import org.openrs2.archive.ArchiveModule
import org.openrs2.cache.Store import org.openrs2.cache.Store
import org.openrs2.cli.instant
public class ImportCommand : CliktCommand(name = "import") { public class ImportCommand : CliktCommand(name = "import") {
private val build by option().int()
private val timestamp by option().instant()
private val game by argument()
private val input by argument().path( private val input by argument().path(
mustExist = true, mustExist = true,
canBeFile = false, canBeFile = false,
@ -20,7 +27,7 @@ public class ImportCommand : CliktCommand(name = "import") {
val importer = injector.getInstance(CacheImporter::class.java) val importer = injector.getInstance(CacheImporter::class.java)
Store.open(input).use { store -> Store.open(input).use { store ->
importer.import(store) importer.import(store, game, build, timestamp)
} }
} }
} }

@ -2,15 +2,22 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice import com.google.inject.Guice
import io.netty.buffer.Unpooled import io.netty.buffer.Unpooled
import kotlinx.coroutines.runBlocking import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule import org.openrs2.archive.ArchiveModule
import org.openrs2.buffer.use import org.openrs2.buffer.use
import org.openrs2.cli.instant
import java.nio.file.Files import java.nio.file.Files
public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index") { public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index") {
private val build by option().int()
private val timestamp by option().instant()
private val game by argument()
private val input by argument().path( private val input by argument().path(
mustExist = true, mustExist = true,
canBeDir = false, canBeDir = false,
@ -22,7 +29,7 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
val importer = injector.getInstance(CacheImporter::class.java) val importer = injector.getInstance(CacheImporter::class.java)
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
importer.importMasterIndex(buf) importer.importMasterIndex(buf, game, build, timestamp)
} }
} }
} }

@ -22,6 +22,7 @@ import org.openrs2.protocol.js5.Js5ResponseDecoder
import org.openrs2.protocol.js5.XorDecoder import org.openrs2.protocol.js5.XorDecoder
import org.openrs2.protocol.login.LoginRequest import org.openrs2.protocol.login.LoginRequest
import org.openrs2.protocol.login.LoginResponse import org.openrs2.protocol.login.LoginResponse
import java.time.Instant
import kotlin.coroutines.Continuation import kotlin.coroutines.Continuation
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@ -156,7 +157,9 @@ public class Js5ChannelHandler(
Js5MasterIndex.read(uncompressed) Js5MasterIndex.read(uncompressed)
} }
val rawIndexes = runBlocking { importer.importMasterIndexAndGetIndexes(masterIndex!!, buf, gameId, build) } val rawIndexes = runBlocking {
importer.importMasterIndexAndGetIndexes(masterIndex!!, buf, gameId, build, Instant.now())
}
try { try {
indexes = arrayOfNulls(rawIndexes.size) indexes = arrayOfNulls(rawIndexes.size)

@ -80,7 +80,10 @@ CREATE TABLE index_files (
); );
CREATE TABLE master_indexes ( CREATE TABLE master_indexes (
container_id BIGINT PRIMARY KEY NOT NULL REFERENCES containers (id) container_id BIGINT PRIMARY KEY NOT NULL REFERENCES containers (id),
game_id INTEGER NOT NULL REFERENCES games (id),
build INTEGER NULL,
timestamp TIMESTAMPTZ NULL
); );
CREATE TABLE master_index_entries ( CREATE TABLE master_index_entries (

Loading…
Cancel
Save