Add environment and language columns to the caches table

Signed-off-by: Graham <gpe@openrs2.org>

archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt

@@ -20,8 +20,8 @@ public class CacheDownloader @Inject constructor(
     private val gameDatabase: GameDatabase,
     private val importer: CacheImporter
 ) {
-    public suspend fun download(gameName: String) {
+    public suspend fun download(gameName: String, environment: String, language: String) {
-        val game = gameDatabase.getGame(gameName) ?: throw Exception("Game not found")
+        val game = gameDatabase.getGame(gameName, environment, language) ?: throw Exception("Game not found")
 
         val url = game.url ?: throw Exception("URL not set")
         val buildMajor = game.buildMajor ?: throw Exception("Current major build not set")
@@ -75,6 +75,7 @@ public class CacheDownloader @Inject constructor(
                                 continuation,
                                 importer,
                                 token,
+                                game.languageId,
                                 musicStreamClient
                             )
                         )

archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt

@@ -99,6 +99,8 @@ public class CacheExporter @Inject constructor(
     public data class CacheSummary(
         val id: Int,
         val game: String,
+        val environment: String,
+        val language: String,
         val builds: SortedSet<Build>,
         val timestamp: Instant?,
         val names: SortedSet<String>,
@@ -116,6 +118,8 @@ public class CacheExporter @Inject constructor(
 
     public data class Source(
         val game: String,
+        val environment: String,
+        val language: String,
         val build: Build?,
         val timestamp: Instant?,
         val name: String?,
@@ -140,7 +144,9 @@ public class CacheExporter @Inject constructor(
                 FROM (
                     SELECT
                         c.id,
-                        g.name,
+                        g.name AS game,
+                        e.name AS environment,
+                        l.iso_code AS language,
                         array_remove(array_agg(DISTINCT ROW(s.build_major, s.build_minor)::build ORDER BY ROW(s.build_major, s.build_minor)::build ASC), NULL) builds,
                         MIN(s.timestamp) AS timestamp,
                         array_remove(array_agg(DISTINCT s.name ORDER BY s.name ASC), NULL) sources,
@@ -154,12 +160,15 @@ public class CacheExporter @Inject constructor(
                         cs.blocks
                     FROM caches c
                     JOIN sources s ON s.cache_id = c.id
-                    JOIN games g ON g.id = s.game_id
+                    JOIN game_variants v ON v.id = s.game_id
+                    JOIN games g ON g.id = v.game_id
+                    JOIN environments e ON e.id = v.environment_id
+                    JOIN languages l ON l.id = v.language_id
                     LEFT JOIN cache_stats cs ON cs.cache_id = c.id
-                    GROUP BY c.id, g.name, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups, cs.valid_keys, cs.keys,
+                    GROUP BY c.id, g.name, e.name, l.iso_code, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups,
-                        cs.size, cs.blocks
+                        cs.valid_keys, cs.keys, cs.size, cs.blocks
                 ) t
-                ORDER BY t.name ASC, t.builds[1] ASC, t.timestamp ASC
+                ORDER BY t.game ASC, t.environment ASC, t.language ASC, t.builds[1] ASC, t.timestamp ASC
             """.trimIndent()
             ).use { stmt ->
                 stmt.executeQuery().use { rows ->
@@ -168,19 +177,21 @@ public class CacheExporter @Inject constructor(
                     while (rows.next()) {
                         val id = rows.getInt(1)
                         val game = rows.getString(2)
-                        val builds = rows.getArray(3).array as Array<*>
+                        val environment = rows.getString(3)
-                        val timestamp = rows.getTimestamp(4)?.toInstant()
+                        val language = rows.getString(4)
-                        @Suppress("UNCHECKED_CAST") val names = rows.getArray(5).array as Array<String>
+                        val builds = rows.getArray(5).array as Array<*>
+                        val timestamp = rows.getTimestamp(6)?.toInstant()
+                        @Suppress("UNCHECKED_CAST") val names = rows.getArray(7).array as Array<String>
 
-                        val validIndexes = rows.getLong(6)
+                        val validIndexes = rows.getLong(8)
                         val stats = if (!rows.wasNull()) {
-                            val indexes = rows.getLong(7)
+                            val indexes = rows.getLong(9)
-                            val validGroups = rows.getLong(8)
+                            val validGroups = rows.getLong(10)
-                            val groups = rows.getLong(9)
+                            val groups = rows.getLong(11)
-                            val validKeys = rows.getLong(10)
+                            val validKeys = rows.getLong(12)
-                            val keys = rows.getLong(11)
+                            val keys = rows.getLong(13)
-                            val size = rows.getLong(12)
+                            val size = rows.getLong(14)
-                            val blocks = rows.getLong(13)
+                            val blocks = rows.getLong(15)
                             Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks)
                         } else {
                             null
@@ -189,6 +200,8 @@ public class CacheExporter @Inject constructor(
                         caches += CacheSummary(
                             id,
                             game,
+                            environment,
+                            language,
                             builds.mapNotNull { o -> Build.fromPgObject(o as PGobject) }.toSortedSet(),
                             timestamp,
                             names.toSortedSet(),
@@ -279,9 +292,12 @@ public class CacheExporter @Inject constructor(
 
             connection.prepareStatement(
                 """
-                SELECT g.name, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url
+                SELECT g.name, e.name, l.iso_code, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url
                 FROM sources s
-                JOIN games g ON g.id = s.game_id
+                JOIN game_variants v ON v.id = s.game_id
+                JOIN games g ON g.id = v.game_id
+                JOIN environments e ON e.id = v.environment_id
+                JOIN languages l ON l.id = v.language_id
                 WHERE s.cache_id = ?
                 ORDER BY s.name ASC
             """.trimIndent()
@@ -291,13 +307,15 @@ public class CacheExporter @Inject constructor(
                 stmt.executeQuery().use { rows ->
                     while (rows.next()) {
                         val game = rows.getString(1)
+                        val environment = rows.getString(2)
+                        val language = rows.getString(3)
 
-                        var buildMajor: Int? = rows.getInt(2)
+                        var buildMajor: Int? = rows.getInt(4)
                         if (rows.wasNull()) {
                             buildMajor = null
                         }
 
-                        var buildMinor: Int? = rows.getInt(3)
+                        var buildMinor: Int? = rows.getInt(5)
                         if (rows.wasNull()) {
                             buildMinor = null
                         }
@@ -308,12 +326,12 @@ public class CacheExporter @Inject constructor(
                             null
                         }
 
-                        val timestamp = rows.getTimestamp(4)?.toInstant()
+                        val timestamp = rows.getTimestamp(6)?.toInstant()
-                        val name = rows.getString(5)
+                        val name = rows.getString(7)
-                        val description = rows.getString(6)
+                        val description = rows.getString(8)
-                        val url = rows.getString(7)
+                        val url = rows.getString(9)
 
-                        sources += Source(game, build, timestamp, name, description, url)
+                        sources += Source(game, environment, language, build, timestamp, name, description, url)
                     }
                 }
             }

archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt

@@ -119,6 +119,8 @@ public class CacheImporter @Inject constructor(
     public suspend fun import(
         store: Store,
         game: String,
+        environment: String,
+        language: String,
         buildMajor: Int?,
         buildMinor: Int?,
         timestamp: Instant?,
@@ -129,7 +131,7 @@ public class CacheImporter @Inject constructor(
         database.execute { connection ->
             prepare(connection)
 
-            val gameId = getGameId(connection, game)
+            val gameId = getGameId(connection, game, environment, language)
 
             if (store is DiskStore && store.legacy) {
                 importLegacy(connection, store, gameId, buildMajor, buildMinor, timestamp, name, description, url)
@@ -233,6 +235,8 @@ public class CacheImporter @Inject constructor(
         buf: ByteBuf,
         format: MasterIndexFormat,
         game: String,
+        environment: String,
+        language: String,
         buildMajor: Int?,
         buildMinor: Int?,
         timestamp: Instant?,
@@ -250,7 +254,7 @@ public class CacheImporter @Inject constructor(
             database.execute { connection ->
                 prepare(connection)
 
-                val gameId = getGameId(connection, game)
+                val gameId = getGameId(connection, game, environment, language)
                 val masterIndexId = addMasterIndex(connection, masterIndex)
 
                 addSource(
@@ -284,7 +288,7 @@ public class CacheImporter @Inject constructor(
 
             connection.prepareStatement(
                 """
-                UPDATE games
+                UPDATE game_variants
                 SET build_major = ?, build_minor = ?
                 WHERE id = ?
             """.trimIndent()
@@ -960,15 +964,20 @@ public class CacheImporter @Inject constructor(
         return ids
     }
 
-    private fun getGameId(connection: Connection, name: String): Int {
+    private fun getGameId(connection: Connection, name: String, environment: String, language: String): Int {
         connection.prepareStatement(
             """
-                SELECT id
+                SELECT v.id
-                FROM games
+                FROM game_variants v
-                WHERE name = ?
+                JOIN games g ON g.id = v.game_id
+                JOIN environments e ON e.id = v.environment_id
+                JOIN languages l ON l.id = v.language_id
+                WHERE g.name = ? AND e.name = ? AND l.iso_code = ?
             """.trimIndent()
         ).use { stmt ->
             stmt.setString(1, name)
+            stmt.setString(2, environment)
+            stmt.setString(3, language)
 
             stmt.executeQuery().use { rows ->
                 if (!rows.next()) {
@@ -984,7 +993,7 @@ public class CacheImporter @Inject constructor(
         database.execute { connection ->
             connection.prepareStatement(
                 """
-                UPDATE games SET last_master_index_id = ? WHERE id = ?
+                UPDATE game_variants SET last_master_index_id = ? WHERE id = ?
             """.trimIndent()
             ).use { stmt ->
                 stmt.setInt(1, masterIndexId)

archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt

@@ -3,18 +3,23 @@ package org.openrs2.archive.cache
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
 import com.github.ajalt.clikt.parameters.arguments.default
+import com.github.ajalt.clikt.parameters.options.default
+import com.github.ajalt.clikt.parameters.options.option
 import com.google.inject.Guice
 import kotlinx.coroutines.runBlocking
 import org.openrs2.archive.ArchiveModule
 import org.openrs2.inject.CloseableInjector
 
 public class DownloadCommand : CliktCommand(name = "download") {
+    private val environment by option().default("live")
+    private val language by option().default("en")
+
     private val game by argument().default("oldschool")
 
     override fun run(): Unit = runBlocking {
         CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector ->
             val downloader = injector.getInstance(CacheDownloader::class.java)
-            downloader.download(game)
+            downloader.download(game, environment, language)
         }
     }
 }

archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt

@@ -2,6 +2,7 @@ package org.openrs2.archive.cache
 
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.options.default
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.path
@@ -19,6 +20,8 @@ public class ImportCommand : CliktCommand(name = "import") {
     private val name by option()
     private val description by option()
     private val url by option()
+    private val environment by option().default("live")
+    private val language by option().default("en")
 
     private val game by argument()
     private val input by argument().path(
@@ -32,7 +35,18 @@ public class ImportCommand : CliktCommand(name = "import") {
             val importer = injector.getInstance(CacheImporter::class.java)
 
             Store.open(input).use { store ->
-                importer.import(store, game, buildMajor, buildMinor, timestamp, name, description, url)
+                importer.import(
+                    store,
+                    game,
+                    environment,
+                    language,
+                    buildMajor,
+                    buildMinor,
+                    timestamp,
+                    name,
+                    description,
+                    url
+                )
             }
         }
     }

archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt

@@ -2,6 +2,7 @@ package org.openrs2.archive.cache
 
 import com.github.ajalt.clikt.core.CliktCommand
 import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.options.default
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.int
@@ -23,6 +24,8 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
     private val name by option()
     private val description by option()
     private val url by option()
+    private val environment by option().default("live")
+    private val language by option().default("en")
 
     private val game by argument()
     private val format by argument().enum<MasterIndexFormat>()
@@ -37,7 +40,19 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
             val importer = injector.getInstance(CacheImporter::class.java)
 
             Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
-                importer.importMasterIndex(buf, format, game, buildMajor, buildMinor, timestamp, name, description, url)
+                importer.importMasterIndex(
+                    buf,
+                    format,
+                    game,
+                    environment,
+                    language,
+                    buildMajor,
+                    buildMinor,
+                    timestamp,
+                    name,
+                    description,
+                    url
+                )
             }
         }
     }

archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt

@@ -33,6 +33,7 @@ public class NxtJs5ChannelHandler(
     continuation: Continuation<Unit>,
     importer: CacheImporter,
     private val token: String,
+    private val languageId: Int,
     private val musicStreamClient: MusicStreamClient,
     private val maxMinorBuildAttempts: Int = 5
 ) : Js5ChannelHandler(
@@ -56,7 +57,7 @@ public class NxtJs5ChannelHandler(
     private var minorBuildAttempts = 0
 
     override fun createInitMessage(): Any {
-        return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, 0)
+        return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, languageId)
     }
 
     override fun createRequestMessage(prefetch: Boolean, archive: Int, group: Int): Any {

archive/src/main/kotlin/org/openrs2/archive/game/Game.kt

@@ -5,5 +5,6 @@ public data class Game(
     public val url: String?,
     public val buildMajor: Int?,
     public val buildMinor: Int?,
-    public val lastMasterIndexId: Int?
+    public val lastMasterIndexId: Int?,
+    public val languageId: Int
 )

archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt

@@ -8,16 +8,21 @@ import javax.inject.Singleton
 public class GameDatabase @Inject constructor(
     private val database: Database
 ) {
-    public suspend fun getGame(name: String): Game? {
+    public suspend fun getGame(name: String, environment: String, language: String): Game? {
         return database.execute { connection ->
             connection.prepareStatement(
                 """
-                SELECT id, url, build_major, build_minor, last_master_index_id
+                SELECT v.id, v.url, v.build_major, v.build_minor, v.last_master_index_id, v.language_id
-                FROM games
+                FROM game_variants v
-                WHERE name = ?
+                JOIN games g ON g.id = v.game_id
+                JOIN environments e ON e.id = v.environment_id
+                JOIN languages l ON l.id = v.language_id
+                WHERE g.name = ? AND e.name = ? AND l.iso_code = ?
             """.trimIndent()
             ).use { stmt ->
                 stmt.setString(1, name)
+                stmt.setString(2, environment)
+                stmt.setString(3, language)
 
                 stmt.executeQuery().use { rows ->
                     if (!rows.next()) {
@@ -42,7 +47,9 @@ public class GameDatabase @Inject constructor(
                         lastMasterIndexId = null
                     }
 
-                    return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId)
+                    val languageId = rows.getInt(6)
+
+                    return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId, languageId)
                 }
             }
         }

archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql

@@ -0,0 +1,59 @@
+-- @formatter:off
+CREATE TABLE environments (
+    id INTEGER NOT NULL PRIMARY KEY,
+    name TEXT NOT NULL
+);
+
+INSERT INTO environments (id, name)
+VALUES
+    (1, 'live'),
+    (2, 'beta');
+
+CREATE TABLE languages (
+    -- Not SERIAL as these IDs are allocated by Jagex, not us.
+    id INTEGER NOT NULL PRIMARY KEY,
+    iso_code TEXT NOT NULL
+);
+
+INSERT INTO languages (id, iso_code)
+VALUES
+    (0, 'en'),
+    (1, 'de'),
+    (2, 'fr'),
+    (3, 'pt');
+
+ALTER TABLE games RENAME TO game_variants;
+
+ALTER INDEX games_pkey RENAME TO game_variants_pkey;
+ALTER INDEX games_name_key RENAME TO game_variants_name_key;
+
+ALTER SEQUENCE games_id_seq RENAME TO game_variants_id_seq;
+
+CREATE TABLE games (
+    id SERIAL NOT NULL PRIMARY KEY,
+    name TEXT UNIQUE NOT NULL
+);
+
+INSERT INTO games (id, name)
+SELECT id, name
+FROM game_variants;
+
+SELECT setval('game_variants_id_seq', MAX(id)) FROM game_variants;
+
+ALTER TABLE game_variants
+    ADD COLUMN game_id INT NULL REFERENCES games (id),
+    ADD COLUMN environment_id INT NOT NULL DEFAULT 1 REFERENCES environments (id),
+    ADD COLUMN language_id INT NOT NULL DEFAULT 0 REFERENCES languages (id);
+
+UPDATE game_variants v
+SET game_id = g.id, environment_id = 1, language_id = 0
+FROM games g
+WHERE g.name = v.name;
+
+ALTER TABLE game_variants
+    DROP COLUMN name,
+    ALTER COLUMN game_id SET NOT NULL,
+    ALTER COLUMN environment_id DROP DEFAULT,
+    ALTER COLUMN language_id DROP DEFAULT;
+
+CREATE UNIQUE INDEX ON game_variants (game_id, environment_id, language_id);

archive/src/main/resources/org/openrs2/archive/templates/caches/index.html

@@ -16,6 +16,8 @@
 					<thead class="table-dark">
 						<tr>
 							<th data-field="game" data-filter-control="select">Game</th>
+							<th data-field="environment" data-filter-control="select">Env</th>
+							<th data-field="language" data-filter-control="select">Lang</th>
 							<th data-field="builds" data-filter-control="input" data-sortable="true">Build(s)</th>
 							<th data-field="timestamp" data-sortable="true">Timestamp</th>
 							<th data-field="sources" data-filter-control="input">Source(s)</th>
@@ -30,6 +32,8 @@
 						<!--/*@thymesVar id="caches" type="java.util.List<org.openrs2.archive.cache.CacheExporter.Cache>"*/-->
 						<tr th:each="cache : ${caches}">
 							<td th:text="${cache.game}">runescape</td>
+							<td th:text="${cache.environment}">live</td>
+							<td th:text="${cache.language}">en</td>
 							<td class="text-right">
 								<span th:each="build, it : ${cache.builds}" th:remove="tag">
 									<span th:text="${build}">550</span>

archive/src/main/resources/org/openrs2/archive/templates/caches/show.html

@@ -88,6 +88,8 @@
 					<thead class="table-dark">
 						<tr>
 							<th>Game</th>
+							<th>Environment</th>
+							<th>Language</th>
 							<th>Build</th>
 							<th>Timestamp</th>
 							<th>Name</th>
@@ -98,6 +100,8 @@
 					<tbody>
 						<tr th:each="source : ${cache.sources}">
 							<td th:text="${source.game}">runescape</td>
+							<td th:text="${source.environment}">live</td>
+							<td th:text="${source.language}">en</td>
 							<td th:text="${source.build}" class="text-right">550</td>
 							<td th:text="${#temporals.format(source.timestamp, 'yyyy-MM-dd HH:mm:ss')}"></td>
 							<td th:text="${source.name}"></td>