From 469fe2eecc9554a04955d214b06d14763180c23a Mon Sep 17 00:00:00 2001 From: Graham Date: Fri, 24 Dec 2021 15:26:18 +0000 Subject: [PATCH] Add environment and language columns to the caches table Signed-off-by: Graham --- .../openrs2/archive/cache/CacheDownloader.kt | 5 +- .../openrs2/archive/cache/CacheExporter.kt | 68 ++++++++++++------- .../openrs2/archive/cache/CacheImporter.kt | 25 ++++--- .../openrs2/archive/cache/DownloadCommand.kt | 7 +- .../openrs2/archive/cache/ImportCommand.kt | 16 ++++- .../archive/cache/ImportMasterIndexCommand.kt | 17 ++++- .../archive/cache/NxtJs5ChannelHandler.kt | 3 +- .../kotlin/org/openrs2/archive/game/Game.kt | 3 +- .../org/openrs2/archive/game/GameDatabase.kt | 17 +++-- .../archive/migrations/V10__variants.sql | 59 ++++++++++++++++ .../archive/templates/caches/index.html | 4 ++ .../archive/templates/caches/show.html | 4 ++ 12 files changed, 183 insertions(+), 45 deletions(-) create mode 100644 archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt index 88b1ee50..39190a16 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt @@ -20,8 +20,8 @@ public class CacheDownloader @Inject constructor( private val gameDatabase: GameDatabase, private val importer: CacheImporter ) { - public suspend fun download(gameName: String) { - val game = gameDatabase.getGame(gameName) ?: throw Exception("Game not found") + public suspend fun download(gameName: String, environment: String, language: String) { + val game = gameDatabase.getGame(gameName, environment, language) ?: throw Exception("Game not found") val url = game.url ?: throw Exception("URL not set") val buildMajor = game.buildMajor ?: throw Exception("Current major build not set") @@ -75,6 +75,7 @@ public class CacheDownloader @Inject constructor( continuation, importer, token, + game.languageId, musicStreamClient ) ) diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt index 0ac4ea13..c19bdcff 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt @@ -99,6 +99,8 @@ public class CacheExporter @Inject constructor( public data class CacheSummary( val id: Int, val game: String, + val environment: String, + val language: String, val builds: SortedSet, val timestamp: Instant?, val names: SortedSet, @@ -116,6 +118,8 @@ public class CacheExporter @Inject constructor( public data class Source( val game: String, + val environment: String, + val language: String, val build: Build?, val timestamp: Instant?, val name: String?, @@ -140,7 +144,9 @@ public class CacheExporter @Inject constructor( FROM ( SELECT c.id, - g.name, + g.name AS game, + e.name AS environment, + l.iso_code AS language, array_remove(array_agg(DISTINCT ROW(s.build_major, s.build_minor)::build ORDER BY ROW(s.build_major, s.build_minor)::build ASC), NULL) builds, MIN(s.timestamp) AS timestamp, array_remove(array_agg(DISTINCT s.name ORDER BY s.name ASC), NULL) sources, @@ -154,12 +160,15 @@ public class CacheExporter @Inject constructor( cs.blocks FROM caches c JOIN sources s ON s.cache_id = c.id - JOIN games g ON g.id = s.game_id + JOIN game_variants v ON v.id = s.game_id + JOIN games g ON g.id = v.game_id + JOIN environments e ON e.id = v.environment_id + JOIN languages l ON l.id = v.language_id LEFT JOIN cache_stats cs ON cs.cache_id = c.id - GROUP BY c.id, g.name, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups, cs.valid_keys, cs.keys, - cs.size, cs.blocks + GROUP BY c.id, g.name, e.name, l.iso_code, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups, + cs.valid_keys, cs.keys, cs.size, cs.blocks ) t - ORDER BY t.name ASC, t.builds[1] ASC, t.timestamp ASC + ORDER BY t.game ASC, t.environment ASC, t.language ASC, t.builds[1] ASC, t.timestamp ASC """.trimIndent() ).use { stmt -> stmt.executeQuery().use { rows -> @@ -168,19 +177,21 @@ public class CacheExporter @Inject constructor( while (rows.next()) { val id = rows.getInt(1) val game = rows.getString(2) - val builds = rows.getArray(3).array as Array<*> - val timestamp = rows.getTimestamp(4)?.toInstant() - @Suppress("UNCHECKED_CAST") val names = rows.getArray(5).array as Array + val environment = rows.getString(3) + val language = rows.getString(4) + val builds = rows.getArray(5).array as Array<*> + val timestamp = rows.getTimestamp(6)?.toInstant() + @Suppress("UNCHECKED_CAST") val names = rows.getArray(7).array as Array - val validIndexes = rows.getLong(6) + val validIndexes = rows.getLong(8) val stats = if (!rows.wasNull()) { - val indexes = rows.getLong(7) - val validGroups = rows.getLong(8) - val groups = rows.getLong(9) - val validKeys = rows.getLong(10) - val keys = rows.getLong(11) - val size = rows.getLong(12) - val blocks = rows.getLong(13) + val indexes = rows.getLong(9) + val validGroups = rows.getLong(10) + val groups = rows.getLong(11) + val validKeys = rows.getLong(12) + val keys = rows.getLong(13) + val size = rows.getLong(14) + val blocks = rows.getLong(15) Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks) } else { null @@ -189,6 +200,8 @@ public class CacheExporter @Inject constructor( caches += CacheSummary( id, game, + environment, + language, builds.mapNotNull { o -> Build.fromPgObject(o as PGobject) }.toSortedSet(), timestamp, names.toSortedSet(), @@ -279,9 +292,12 @@ public class CacheExporter @Inject constructor( connection.prepareStatement( """ - SELECT g.name, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url + SELECT g.name, e.name, l.iso_code, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url FROM sources s - JOIN games g ON g.id = s.game_id + JOIN game_variants v ON v.id = s.game_id + JOIN games g ON g.id = v.game_id + JOIN environments e ON e.id = v.environment_id + JOIN languages l ON l.id = v.language_id WHERE s.cache_id = ? ORDER BY s.name ASC """.trimIndent() @@ -291,13 +307,15 @@ public class CacheExporter @Inject constructor( stmt.executeQuery().use { rows -> while (rows.next()) { val game = rows.getString(1) + val environment = rows.getString(2) + val language = rows.getString(3) - var buildMajor: Int? = rows.getInt(2) + var buildMajor: Int? = rows.getInt(4) if (rows.wasNull()) { buildMajor = null } - var buildMinor: Int? = rows.getInt(3) + var buildMinor: Int? = rows.getInt(5) if (rows.wasNull()) { buildMinor = null } @@ -308,12 +326,12 @@ public class CacheExporter @Inject constructor( null } - val timestamp = rows.getTimestamp(4)?.toInstant() - val name = rows.getString(5) - val description = rows.getString(6) - val url = rows.getString(7) + val timestamp = rows.getTimestamp(6)?.toInstant() + val name = rows.getString(7) + val description = rows.getString(8) + val url = rows.getString(9) - sources += Source(game, build, timestamp, name, description, url) + sources += Source(game, environment, language, build, timestamp, name, description, url) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index 83c537ce..2d4d09d2 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -119,6 +119,8 @@ public class CacheImporter @Inject constructor( public suspend fun import( store: Store, game: String, + environment: String, + language: String, buildMajor: Int?, buildMinor: Int?, timestamp: Instant?, @@ -129,7 +131,7 @@ public class CacheImporter @Inject constructor( database.execute { connection -> prepare(connection) - val gameId = getGameId(connection, game) + val gameId = getGameId(connection, game, environment, language) if (store is DiskStore && store.legacy) { importLegacy(connection, store, gameId, buildMajor, buildMinor, timestamp, name, description, url) @@ -233,6 +235,8 @@ public class CacheImporter @Inject constructor( buf: ByteBuf, format: MasterIndexFormat, game: String, + environment: String, + language: String, buildMajor: Int?, buildMinor: Int?, timestamp: Instant?, @@ -250,7 +254,7 @@ public class CacheImporter @Inject constructor( database.execute { connection -> prepare(connection) - val gameId = getGameId(connection, game) + val gameId = getGameId(connection, game, environment, language) val masterIndexId = addMasterIndex(connection, masterIndex) addSource( @@ -284,7 +288,7 @@ public class CacheImporter @Inject constructor( connection.prepareStatement( """ - UPDATE games + UPDATE game_variants SET build_major = ?, build_minor = ? WHERE id = ? """.trimIndent() @@ -960,15 +964,20 @@ public class CacheImporter @Inject constructor( return ids } - private fun getGameId(connection: Connection, name: String): Int { + private fun getGameId(connection: Connection, name: String, environment: String, language: String): Int { connection.prepareStatement( """ - SELECT id - FROM games - WHERE name = ? + SELECT v.id + FROM game_variants v + JOIN games g ON g.id = v.game_id + JOIN environments e ON e.id = v.environment_id + JOIN languages l ON l.id = v.language_id + WHERE g.name = ? AND e.name = ? AND l.iso_code = ? """.trimIndent() ).use { stmt -> stmt.setString(1, name) + stmt.setString(2, environment) + stmt.setString(3, language) stmt.executeQuery().use { rows -> if (!rows.next()) { @@ -984,7 +993,7 @@ public class CacheImporter @Inject constructor( database.execute { connection -> connection.prepareStatement( """ - UPDATE games SET last_master_index_id = ? WHERE id = ? + UPDATE game_variants SET last_master_index_id = ? WHERE id = ? """.trimIndent() ).use { stmt -> stmt.setInt(1, masterIndexId) diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt index 9b01ffcb..e97dc7c5 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt @@ -3,18 +3,23 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.default +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.option import com.google.inject.Guice import kotlinx.coroutines.runBlocking import org.openrs2.archive.ArchiveModule import org.openrs2.inject.CloseableInjector public class DownloadCommand : CliktCommand(name = "download") { + private val environment by option().default("live") + private val language by option().default("en") + private val game by argument().default("oldschool") override fun run(): Unit = runBlocking { CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> val downloader = injector.getInstance(CacheDownloader::class.java) - downloader.download(game) + downloader.download(game, environment, language) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt index df9acd30..52bdf7a9 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt @@ -2,6 +2,7 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.default import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.path @@ -19,6 +20,8 @@ public class ImportCommand : CliktCommand(name = "import") { private val name by option() private val description by option() private val url by option() + private val environment by option().default("live") + private val language by option().default("en") private val game by argument() private val input by argument().path( @@ -32,7 +35,18 @@ public class ImportCommand : CliktCommand(name = "import") { val importer = injector.getInstance(CacheImporter::class.java) Store.open(input).use { store -> - importer.import(store, game, buildMajor, buildMinor, timestamp, name, description, url) + importer.import( + store, + game, + environment, + language, + buildMajor, + buildMinor, + timestamp, + name, + description, + url + ) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt index 1f4249ab..d47bcf5b 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt @@ -2,6 +2,7 @@ package org.openrs2.archive.cache import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.options.default import com.github.ajalt.clikt.parameters.options.option import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.int @@ -23,6 +24,8 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index private val name by option() private val description by option() private val url by option() + private val environment by option().default("live") + private val language by option().default("en") private val game by argument() private val format by argument().enum() @@ -37,7 +40,19 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index val importer = injector.getInstance(CacheImporter::class.java) Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> - importer.importMasterIndex(buf, format, game, buildMajor, buildMinor, timestamp, name, description, url) + importer.importMasterIndex( + buf, + format, + game, + environment, + language, + buildMajor, + buildMinor, + timestamp, + name, + description, + url + ) } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt index faa0fc28..d6c705c8 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt @@ -33,6 +33,7 @@ public class NxtJs5ChannelHandler( continuation: Continuation, importer: CacheImporter, private val token: String, + private val languageId: Int, private val musicStreamClient: MusicStreamClient, private val maxMinorBuildAttempts: Int = 5 ) : Js5ChannelHandler( @@ -56,7 +57,7 @@ public class NxtJs5ChannelHandler( private var minorBuildAttempts = 0 override fun createInitMessage(): Any { - return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, 0) + return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, languageId) } override fun createRequestMessage(prefetch: Boolean, archive: Int, group: Int): Any { diff --git a/archive/src/main/kotlin/org/openrs2/archive/game/Game.kt b/archive/src/main/kotlin/org/openrs2/archive/game/Game.kt index 0ddfcbbf..9bcd54f8 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/game/Game.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/game/Game.kt @@ -5,5 +5,6 @@ public data class Game( public val url: String?, public val buildMajor: Int?, public val buildMinor: Int?, - public val lastMasterIndexId: Int? + public val lastMasterIndexId: Int?, + public val languageId: Int ) diff --git a/archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt b/archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt index 318a81a6..da1fdb6b 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt @@ -8,16 +8,21 @@ import javax.inject.Singleton public class GameDatabase @Inject constructor( private val database: Database ) { - public suspend fun getGame(name: String): Game? { + public suspend fun getGame(name: String, environment: String, language: String): Game? { return database.execute { connection -> connection.prepareStatement( """ - SELECT id, url, build_major, build_minor, last_master_index_id - FROM games - WHERE name = ? + SELECT v.id, v.url, v.build_major, v.build_minor, v.last_master_index_id, v.language_id + FROM game_variants v + JOIN games g ON g.id = v.game_id + JOIN environments e ON e.id = v.environment_id + JOIN languages l ON l.id = v.language_id + WHERE g.name = ? AND e.name = ? AND l.iso_code = ? """.trimIndent() ).use { stmt -> stmt.setString(1, name) + stmt.setString(2, environment) + stmt.setString(3, language) stmt.executeQuery().use { rows -> if (!rows.next()) { @@ -42,7 +47,9 @@ public class GameDatabase @Inject constructor( lastMasterIndexId = null } - return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId) + val languageId = rows.getInt(6) + + return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId, languageId) } } } diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql new file mode 100644 index 00000000..f3d6d2ce --- /dev/null +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql @@ -0,0 +1,59 @@ +-- @formatter:off +CREATE TABLE environments ( + id INTEGER NOT NULL PRIMARY KEY, + name TEXT NOT NULL +); + +INSERT INTO environments (id, name) +VALUES + (1, 'live'), + (2, 'beta'); + +CREATE TABLE languages ( + -- Not SERIAL as these IDs are allocated by Jagex, not us. + id INTEGER NOT NULL PRIMARY KEY, + iso_code TEXT NOT NULL +); + +INSERT INTO languages (id, iso_code) +VALUES + (0, 'en'), + (1, 'de'), + (2, 'fr'), + (3, 'pt'); + +ALTER TABLE games RENAME TO game_variants; + +ALTER INDEX games_pkey RENAME TO game_variants_pkey; +ALTER INDEX games_name_key RENAME TO game_variants_name_key; + +ALTER SEQUENCE games_id_seq RENAME TO game_variants_id_seq; + +CREATE TABLE games ( + id SERIAL NOT NULL PRIMARY KEY, + name TEXT UNIQUE NOT NULL +); + +INSERT INTO games (id, name) +SELECT id, name +FROM game_variants; + +SELECT setval('game_variants_id_seq', MAX(id)) FROM game_variants; + +ALTER TABLE game_variants + ADD COLUMN game_id INT NULL REFERENCES games (id), + ADD COLUMN environment_id INT NOT NULL DEFAULT 1 REFERENCES environments (id), + ADD COLUMN language_id INT NOT NULL DEFAULT 0 REFERENCES languages (id); + +UPDATE game_variants v +SET game_id = g.id, environment_id = 1, language_id = 0 +FROM games g +WHERE g.name = v.name; + +ALTER TABLE game_variants + DROP COLUMN name, + ALTER COLUMN game_id SET NOT NULL, + ALTER COLUMN environment_id DROP DEFAULT, + ALTER COLUMN language_id DROP DEFAULT; + +CREATE UNIQUE INDEX ON game_variants (game_id, environment_id, language_id); diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index b64f79ff..b983408f 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -16,6 +16,8 @@ Game + Env + Lang Build(s) Timestamp Source(s) @@ -30,6 +32,8 @@ runescape + live + en 550 diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html index d76e176a..1aa16474 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html @@ -88,6 +88,8 @@ Game + Environment + Language Build Timestamp Name @@ -98,6 +100,8 @@ runescape + live + en 550