Add environment and language columns to the caches table

Signed-off-by: Graham <gpe@openrs2.org>
Graham 3 years ago
parent bc51a68cb3
commit 469fe2eecc
  1. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt
  2. 68
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt
  3. 25
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  4. 7
      archive/src/main/kotlin/org/openrs2/archive/cache/DownloadCommand.kt
  5. 16
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportCommand.kt
  6. 17
      archive/src/main/kotlin/org/openrs2/archive/cache/ImportMasterIndexCommand.kt
  7. 3
      archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt
  8. 3
      archive/src/main/kotlin/org/openrs2/archive/game/Game.kt
  9. 17
      archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt
  10. 59
      archive/src/main/resources/org/openrs2/archive/migrations/V10__variants.sql
  11. 4
      archive/src/main/resources/org/openrs2/archive/templates/caches/index.html
  12. 4
      archive/src/main/resources/org/openrs2/archive/templates/caches/show.html

@ -20,8 +20,8 @@ public class CacheDownloader @Inject constructor(
private val gameDatabase: GameDatabase,
private val importer: CacheImporter
) {
public suspend fun download(gameName: String) {
val game = gameDatabase.getGame(gameName) ?: throw Exception("Game not found")
public suspend fun download(gameName: String, environment: String, language: String) {
val game = gameDatabase.getGame(gameName, environment, language) ?: throw Exception("Game not found")
val url = game.url ?: throw Exception("URL not set")
val buildMajor = game.buildMajor ?: throw Exception("Current major build not set")
@ -75,6 +75,7 @@ public class CacheDownloader @Inject constructor(
continuation,
importer,
token,
game.languageId,
musicStreamClient
)
)

@ -99,6 +99,8 @@ public class CacheExporter @Inject constructor(
public data class CacheSummary(
val id: Int,
val game: String,
val environment: String,
val language: String,
val builds: SortedSet<Build>,
val timestamp: Instant?,
val names: SortedSet<String>,
@ -116,6 +118,8 @@ public class CacheExporter @Inject constructor(
public data class Source(
val game: String,
val environment: String,
val language: String,
val build: Build?,
val timestamp: Instant?,
val name: String?,
@ -140,7 +144,9 @@ public class CacheExporter @Inject constructor(
FROM (
SELECT
c.id,
g.name,
g.name AS game,
e.name AS environment,
l.iso_code AS language,
array_remove(array_agg(DISTINCT ROW(s.build_major, s.build_minor)::build ORDER BY ROW(s.build_major, s.build_minor)::build ASC), NULL) builds,
MIN(s.timestamp) AS timestamp,
array_remove(array_agg(DISTINCT s.name ORDER BY s.name ASC), NULL) sources,
@ -154,12 +160,15 @@ public class CacheExporter @Inject constructor(
cs.blocks
FROM caches c
JOIN sources s ON s.cache_id = c.id
JOIN games g ON g.id = s.game_id
JOIN game_variants v ON v.id = s.game_id
JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id
LEFT JOIN cache_stats cs ON cs.cache_id = c.id
GROUP BY c.id, g.name, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups, cs.valid_keys, cs.keys,
cs.size, cs.blocks
GROUP BY c.id, g.name, e.name, l.iso_code, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups,
cs.valid_keys, cs.keys, cs.size, cs.blocks
) t
ORDER BY t.name ASC, t.builds[1] ASC, t.timestamp ASC
ORDER BY t.game ASC, t.environment ASC, t.language ASC, t.builds[1] ASC, t.timestamp ASC
""".trimIndent()
).use { stmt ->
stmt.executeQuery().use { rows ->
@ -168,19 +177,21 @@ public class CacheExporter @Inject constructor(
while (rows.next()) {
val id = rows.getInt(1)
val game = rows.getString(2)
val builds = rows.getArray(3).array as Array<*>
val timestamp = rows.getTimestamp(4)?.toInstant()
@Suppress("UNCHECKED_CAST") val names = rows.getArray(5).array as Array<String>
val environment = rows.getString(3)
val language = rows.getString(4)
val builds = rows.getArray(5).array as Array<*>
val timestamp = rows.getTimestamp(6)?.toInstant()
@Suppress("UNCHECKED_CAST") val names = rows.getArray(7).array as Array<String>
val validIndexes = rows.getLong(6)
val validIndexes = rows.getLong(8)
val stats = if (!rows.wasNull()) {
val indexes = rows.getLong(7)
val validGroups = rows.getLong(8)
val groups = rows.getLong(9)
val validKeys = rows.getLong(10)
val keys = rows.getLong(11)
val size = rows.getLong(12)
val blocks = rows.getLong(13)
val indexes = rows.getLong(9)
val validGroups = rows.getLong(10)
val groups = rows.getLong(11)
val validKeys = rows.getLong(12)
val keys = rows.getLong(13)
val size = rows.getLong(14)
val blocks = rows.getLong(15)
Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks)
} else {
null
@ -189,6 +200,8 @@ public class CacheExporter @Inject constructor(
caches += CacheSummary(
id,
game,
environment,
language,
builds.mapNotNull { o -> Build.fromPgObject(o as PGobject) }.toSortedSet(),
timestamp,
names.toSortedSet(),
@ -279,9 +292,12 @@ public class CacheExporter @Inject constructor(
connection.prepareStatement(
"""
SELECT g.name, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url
SELECT g.name, e.name, l.iso_code, s.build_major, s.build_minor, s.timestamp, s.name, s.description, s.url
FROM sources s
JOIN games g ON g.id = s.game_id
JOIN game_variants v ON v.id = s.game_id
JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id
WHERE s.cache_id = ?
ORDER BY s.name ASC
""".trimIndent()
@ -291,13 +307,15 @@ public class CacheExporter @Inject constructor(
stmt.executeQuery().use { rows ->
while (rows.next()) {
val game = rows.getString(1)
val environment = rows.getString(2)
val language = rows.getString(3)
var buildMajor: Int? = rows.getInt(2)
var buildMajor: Int? = rows.getInt(4)
if (rows.wasNull()) {
buildMajor = null
}
var buildMinor: Int? = rows.getInt(3)
var buildMinor: Int? = rows.getInt(5)
if (rows.wasNull()) {
buildMinor = null
}
@ -308,12 +326,12 @@ public class CacheExporter @Inject constructor(
null
}
val timestamp = rows.getTimestamp(4)?.toInstant()
val name = rows.getString(5)
val description = rows.getString(6)
val url = rows.getString(7)
val timestamp = rows.getTimestamp(6)?.toInstant()
val name = rows.getString(7)
val description = rows.getString(8)
val url = rows.getString(9)
sources += Source(game, build, timestamp, name, description, url)
sources += Source(game, environment, language, build, timestamp, name, description, url)
}
}
}

@ -119,6 +119,8 @@ public class CacheImporter @Inject constructor(
public suspend fun import(
store: Store,
game: String,
environment: String,
language: String,
buildMajor: Int?,
buildMinor: Int?,
timestamp: Instant?,
@ -129,7 +131,7 @@ public class CacheImporter @Inject constructor(
database.execute { connection ->
prepare(connection)
val gameId = getGameId(connection, game)
val gameId = getGameId(connection, game, environment, language)
if (store is DiskStore && store.legacy) {
importLegacy(connection, store, gameId, buildMajor, buildMinor, timestamp, name, description, url)
@ -233,6 +235,8 @@ public class CacheImporter @Inject constructor(
buf: ByteBuf,
format: MasterIndexFormat,
game: String,
environment: String,
language: String,
buildMajor: Int?,
buildMinor: Int?,
timestamp: Instant?,
@ -250,7 +254,7 @@ public class CacheImporter @Inject constructor(
database.execute { connection ->
prepare(connection)
val gameId = getGameId(connection, game)
val gameId = getGameId(connection, game, environment, language)
val masterIndexId = addMasterIndex(connection, masterIndex)
addSource(
@ -284,7 +288,7 @@ public class CacheImporter @Inject constructor(
connection.prepareStatement(
"""
UPDATE games
UPDATE game_variants
SET build_major = ?, build_minor = ?
WHERE id = ?
""".trimIndent()
@ -960,15 +964,20 @@ public class CacheImporter @Inject constructor(
return ids
}
private fun getGameId(connection: Connection, name: String): Int {
private fun getGameId(connection: Connection, name: String, environment: String, language: String): Int {
connection.prepareStatement(
"""
SELECT id
FROM games
WHERE name = ?
SELECT v.id
FROM game_variants v
JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id
WHERE g.name = ? AND e.name = ? AND l.iso_code = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, name)
stmt.setString(2, environment)
stmt.setString(3, language)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
@ -984,7 +993,7 @@ public class CacheImporter @Inject constructor(
database.execute { connection ->
connection.prepareStatement(
"""
UPDATE games SET last_master_index_id = ? WHERE id = ?
UPDATE game_variants SET last_master_index_id = ? WHERE id = ?
""".trimIndent()
).use { stmt ->
stmt.setInt(1, masterIndexId)

@ -3,18 +3,23 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.arguments.default
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.option
import com.google.inject.Guice
import kotlinx.coroutines.runBlocking
import org.openrs2.archive.ArchiveModule
import org.openrs2.inject.CloseableInjector
public class DownloadCommand : CliktCommand(name = "download") {
private val environment by option().default("live")
private val language by option().default("en")
private val game by argument().default("oldschool")
override fun run(): Unit = runBlocking {
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector ->
val downloader = injector.getInstance(CacheDownloader::class.java)
downloader.download(game)
downloader.download(game, environment, language)
}
}
}

@ -2,6 +2,7 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path
@ -19,6 +20,8 @@ public class ImportCommand : CliktCommand(name = "import") {
private val name by option()
private val description by option()
private val url by option()
private val environment by option().default("live")
private val language by option().default("en")
private val game by argument()
private val input by argument().path(
@ -32,7 +35,18 @@ public class ImportCommand : CliktCommand(name = "import") {
val importer = injector.getInstance(CacheImporter::class.java)
Store.open(input).use { store ->
importer.import(store, game, buildMajor, buildMinor, timestamp, name, description, url)
importer.import(
store,
game,
environment,
language,
buildMajor,
buildMinor,
timestamp,
name,
description,
url
)
}
}
}

@ -2,6 +2,7 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.int
@ -23,6 +24,8 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
private val name by option()
private val description by option()
private val url by option()
private val environment by option().default("live")
private val language by option().default("en")
private val game by argument()
private val format by argument().enum<MasterIndexFormat>()
@ -37,7 +40,19 @@ public class ImportMasterIndexCommand : CliktCommand(name = "import-master-index
val importer = injector.getInstance(CacheImporter::class.java)
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf ->
importer.importMasterIndex(buf, format, game, buildMajor, buildMinor, timestamp, name, description, url)
importer.importMasterIndex(
buf,
format,
game,
environment,
language,
buildMajor,
buildMinor,
timestamp,
name,
description,
url
)
}
}
}

@ -33,6 +33,7 @@ public class NxtJs5ChannelHandler(
continuation: Continuation<Unit>,
importer: CacheImporter,
private val token: String,
private val languageId: Int,
private val musicStreamClient: MusicStreamClient,
private val maxMinorBuildAttempts: Int = 5
) : Js5ChannelHandler(
@ -56,7 +57,7 @@ public class NxtJs5ChannelHandler(
private var minorBuildAttempts = 0
override fun createInitMessage(): Any {
return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, 0)
return InitJs5RemoteConnection(buildMajor, buildMinor!!, token, languageId)
}
override fun createRequestMessage(prefetch: Boolean, archive: Int, group: Int): Any {

@ -5,5 +5,6 @@ public data class Game(
public val url: String?,
public val buildMajor: Int?,
public val buildMinor: Int?,
public val lastMasterIndexId: Int?
public val lastMasterIndexId: Int?,
public val languageId: Int
)

@ -8,16 +8,21 @@ import javax.inject.Singleton
public class GameDatabase @Inject constructor(
private val database: Database
) {
public suspend fun getGame(name: String): Game? {
public suspend fun getGame(name: String, environment: String, language: String): Game? {
return database.execute { connection ->
connection.prepareStatement(
"""
SELECT id, url, build_major, build_minor, last_master_index_id
FROM games
WHERE name = ?
SELECT v.id, v.url, v.build_major, v.build_minor, v.last_master_index_id, v.language_id
FROM game_variants v
JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id
WHERE g.name = ? AND e.name = ? AND l.iso_code = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, name)
stmt.setString(2, environment)
stmt.setString(3, language)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
@ -42,7 +47,9 @@ public class GameDatabase @Inject constructor(
lastMasterIndexId = null
}
return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId)
val languageId = rows.getInt(6)
return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId, languageId)
}
}
}

@ -0,0 +1,59 @@
-- @formatter:off
CREATE TABLE environments (
id INTEGER NOT NULL PRIMARY KEY,
name TEXT NOT NULL
);
INSERT INTO environments (id, name)
VALUES
(1, 'live'),
(2, 'beta');
CREATE TABLE languages (
-- Not SERIAL as these IDs are allocated by Jagex, not us.
id INTEGER NOT NULL PRIMARY KEY,
iso_code TEXT NOT NULL
);
INSERT INTO languages (id, iso_code)
VALUES
(0, 'en'),
(1, 'de'),
(2, 'fr'),
(3, 'pt');
ALTER TABLE games RENAME TO game_variants;
ALTER INDEX games_pkey RENAME TO game_variants_pkey;
ALTER INDEX games_name_key RENAME TO game_variants_name_key;
ALTER SEQUENCE games_id_seq RENAME TO game_variants_id_seq;
CREATE TABLE games (
id SERIAL NOT NULL PRIMARY KEY,
name TEXT UNIQUE NOT NULL
);
INSERT INTO games (id, name)
SELECT id, name
FROM game_variants;
SELECT setval('game_variants_id_seq', MAX(id)) FROM game_variants;
ALTER TABLE game_variants
ADD COLUMN game_id INT NULL REFERENCES games (id),
ADD COLUMN environment_id INT NOT NULL DEFAULT 1 REFERENCES environments (id),
ADD COLUMN language_id INT NOT NULL DEFAULT 0 REFERENCES languages (id);
UPDATE game_variants v
SET game_id = g.id, environment_id = 1, language_id = 0
FROM games g
WHERE g.name = v.name;
ALTER TABLE game_variants
DROP COLUMN name,
ALTER COLUMN game_id SET NOT NULL,
ALTER COLUMN environment_id DROP DEFAULT,
ALTER COLUMN language_id DROP DEFAULT;
CREATE UNIQUE INDEX ON game_variants (game_id, environment_id, language_id);

@ -16,6 +16,8 @@
<thead class="table-dark">
<tr>
<th data-field="game" data-filter-control="select">Game</th>
<th data-field="environment" data-filter-control="select">Env</th>
<th data-field="language" data-filter-control="select">Lang</th>
<th data-field="builds" data-filter-control="input" data-sortable="true">Build(s)</th>
<th data-field="timestamp" data-sortable="true">Timestamp</th>
<th data-field="sources" data-filter-control="input">Source(s)</th>
@ -30,6 +32,8 @@
<!--/*@thymesVar id="caches" type="java.util.List<org.openrs2.archive.cache.CacheExporter.Cache>"*/-->
<tr th:each="cache : ${caches}">
<td th:text="${cache.game}">runescape</td>
<td th:text="${cache.environment}">live</td>
<td th:text="${cache.language}">en</td>
<td class="text-right">
<span th:each="build, it : ${cache.builds}" th:remove="tag">
<span th:text="${build}">550</span>

@ -88,6 +88,8 @@
<thead class="table-dark">
<tr>
<th>Game</th>
<th>Environment</th>
<th>Language</th>
<th>Build</th>
<th>Timestamp</th>
<th>Name</th>
@ -98,6 +100,8 @@
<tbody>
<tr th:each="source : ${cache.sources}">
<td th:text="${source.game}">runescape</td>
<td th:text="${source.environment}">live</td>
<td th:text="${source.language}">en</td>
<td th:text="${source.build}" class="text-right">550</td>
<td th:text="${#temporals.format(source.timestamp, 'yyyy-MM-dd HH:mm:ss')}"></td>
<td th:text="${source.name}"></td>

Loading…
Cancel
Save