Add initial support for separate scopes to the archiving service

This will allow us to import FunOrb caches without worrying about the
risk of collisions with the main set of RuneScape caches.

Signed-off-by: Graham <gpe@openrs2.org>
master
Graham 3 years ago
parent 2c31776c54
commit d186f5aef4
  1. 2
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheDownloader.kt
  2. 85
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt
  3. 70
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  4. 5
      archive/src/main/kotlin/org/openrs2/archive/cache/ExportCommand.kt
  5. 13
      archive/src/main/kotlin/org/openrs2/archive/cache/Js5ChannelHandler.kt
  6. 2
      archive/src/main/kotlin/org/openrs2/archive/cache/NxtJs5ChannelHandler.kt
  7. 2
      archive/src/main/kotlin/org/openrs2/archive/cache/OsrsJs5ChannelHandler.kt
  8. 3
      archive/src/main/kotlin/org/openrs2/archive/game/Game.kt
  9. 5
      archive/src/main/kotlin/org/openrs2/archive/game/GameDatabase.kt
  10. 63
      archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt
  11. 37
      archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt
  12. 61
      archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt
  13. 176
      archive/src/main/resources/org/openrs2/archive/migrations/V12__scopes.sql
  14. 12
      archive/src/main/resources/org/openrs2/archive/templates/caches/index.html
  15. 10
      archive/src/main/resources/org/openrs2/archive/templates/caches/show.html

@ -43,6 +43,7 @@ public class CacheDownloader @Inject constructor(
OsrsJs5ChannelInitializer( OsrsJs5ChannelInitializer(
OsrsJs5ChannelHandler( OsrsJs5ChannelHandler(
bootstrap, bootstrap,
game.scopeId,
game.id, game.id,
hostname, hostname,
PORT, PORT,
@ -66,6 +67,7 @@ public class CacheDownloader @Inject constructor(
NxtJs5ChannelInitializer( NxtJs5ChannelInitializer(
NxtJs5ChannelHandler( NxtJs5ChannelHandler(
bootstrap, bootstrap,
game.scopeId,
game.id, game.id,
hostname, hostname,
PORT, PORT,

@ -111,6 +111,7 @@ public class CacheExporter @Inject constructor(
public data class CacheSummary( public data class CacheSummary(
val id: Int, val id: Int,
val scope: String,
val game: String, val game: String,
val environment: String, val environment: String,
val language: String, val language: String,
@ -159,6 +160,7 @@ public class CacheExporter @Inject constructor(
SELECT SELECT
c.id, c.id,
g.name AS game, g.name AS game,
sc.name AS scope,
e.name AS environment, e.name AS environment,
l.iso_code AS language, l.iso_code AS language,
array_remove(array_agg(DISTINCT ROW(s.build_major, s.build_minor)::build ORDER BY ROW(s.build_major, s.build_minor)::build ASC), NULL) builds, array_remove(array_agg(DISTINCT ROW(s.build_major, s.build_minor)::build ORDER BY ROW(s.build_major, s.build_minor)::build ASC), NULL) builds,
@ -176,11 +178,12 @@ public class CacheExporter @Inject constructor(
JOIN sources s ON s.cache_id = c.id JOIN sources s ON s.cache_id = c.id
JOIN game_variants v ON v.id = s.game_id JOIN game_variants v ON v.id = s.game_id
JOIN games g ON g.id = v.game_id JOIN games g ON g.id = v.game_id
JOIN scopes sc ON sc.id = g.scope_id
JOIN environments e ON e.id = v.environment_id JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id JOIN languages l ON l.id = v.language_id
LEFT JOIN cache_stats cs ON cs.cache_id = c.id LEFT JOIN cache_stats cs ON cs.cache_id = c.id
GROUP BY c.id, g.name, e.name, l.iso_code, cs.valid_indexes, cs.indexes, cs.valid_groups, cs.groups, GROUP BY sc.name, c.id, g.name, e.name, l.iso_code, cs.valid_indexes, cs.indexes, cs.valid_groups,
cs.valid_keys, cs.keys, cs.size, cs.blocks cs.groups, cs.valid_keys, cs.keys, cs.size, cs.blocks
) t ) t
ORDER BY t.game ASC, t.environment ASC, t.language ASC, t.builds[1] ASC, t.timestamp ASC ORDER BY t.game ASC, t.environment ASC, t.language ASC, t.builds[1] ASC, t.timestamp ASC
""".trimIndent() """.trimIndent()
@ -191,21 +194,22 @@ public class CacheExporter @Inject constructor(
while (rows.next()) { while (rows.next()) {
val id = rows.getInt(1) val id = rows.getInt(1)
val game = rows.getString(2) val game = rows.getString(2)
val environment = rows.getString(3) val scope = rows.getString(3)
val language = rows.getString(4) val environment = rows.getString(4)
val builds = rows.getArray(5).array as Array<*> val language = rows.getString(5)
val timestamp = rows.getTimestamp(6)?.toInstant() val builds = rows.getArray(6).array as Array<*>
@Suppress("UNCHECKED_CAST") val sources = rows.getArray(7).array as Array<String> val timestamp = rows.getTimestamp(7)?.toInstant()
@Suppress("UNCHECKED_CAST") val sources = rows.getArray(8).array as Array<String>
val validIndexes = rows.getLong(8)
val validIndexes = rows.getLong(9)
val stats = if (!rows.wasNull()) { val stats = if (!rows.wasNull()) {
val indexes = rows.getLong(9) val indexes = rows.getLong(10)
val validGroups = rows.getLong(10) val validGroups = rows.getLong(11)
val groups = rows.getLong(11) val groups = rows.getLong(12)
val validKeys = rows.getLong(12) val validKeys = rows.getLong(13)
val keys = rows.getLong(13) val keys = rows.getLong(14)
val size = rows.getLong(14) val size = rows.getLong(15)
val blocks = rows.getLong(15) val blocks = rows.getLong(16)
Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks) Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks)
} else { } else {
null null
@ -213,6 +217,7 @@ public class CacheExporter @Inject constructor(
caches += CacheSummary( caches += CacheSummary(
id, id,
scope,
game, game,
environment, environment,
language, language,
@ -229,7 +234,7 @@ public class CacheExporter @Inject constructor(
} }
} }
public suspend fun get(id: Int): Cache? { public suspend fun get(scope: String, id: Int): Cache? {
return database.execute { connection -> return database.execute { connection ->
val masterIndex: Js5MasterIndex? val masterIndex: Js5MasterIndex?
val checksumTable: ChecksumTable? val checksumTable: ChecksumTable?
@ -250,15 +255,17 @@ public class CacheExporter @Inject constructor(
cs.size, cs.size,
cs.blocks cs.blocks
FROM caches c FROM caches c
CROSS JOIN scopes s
LEFT JOIN master_indexes m ON m.id = c.id LEFT JOIN master_indexes m ON m.id = c.id
LEFT JOIN containers mc ON mc.id = m.container_id LEFT JOIN containers mc ON mc.id = m.container_id
LEFT JOIN crc_tables t ON t.id = c.id LEFT JOIN crc_tables t ON t.id = c.id
LEFT JOIN blobs b ON b.id = t.blob_id LEFT JOIN blobs b ON b.id = t.blob_id
LEFT JOIN cache_stats cs ON cs.cache_id = c.id LEFT JOIN cache_stats cs ON cs.scope_id = s.id AND cs.cache_id = c.id
WHERE c.id = ? WHERE s.name = ? AND c.id = ?
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, id) stmt.setString(1, scope)
stmt.setInt(2, id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
if (!rows.next()) { if (!rows.next()) {
@ -310,13 +317,15 @@ public class CacheExporter @Inject constructor(
FROM sources s FROM sources s
JOIN game_variants v ON v.id = s.game_id JOIN game_variants v ON v.id = s.game_id
JOIN games g ON g.id = v.game_id JOIN games g ON g.id = v.game_id
JOIN scopes sc ON sc.id = g.scope_id
JOIN environments e ON e.id = v.environment_id JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id JOIN languages l ON l.id = v.language_id
WHERE s.cache_id = ? WHERE sc.name = ? AND s.cache_id = ?
ORDER BY s.name ASC ORDER BY s.name ASC
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, id) stmt.setString(1, scope)
stmt.setInt(2, id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
while (rows.next()) { while (rows.next()) {
@ -372,7 +381,7 @@ public class CacheExporter @Inject constructor(
} }
} }
public suspend fun getFileName(id: Int): String? { public suspend fun getFileName(scope: String, id: Int): String? {
return database.execute { connection -> return database.execute { connection ->
// TODO(gpe): what if a cache is from multiple games? // TODO(gpe): what if a cache is from multiple games?
connection.prepareStatement( connection.prepareStatement(
@ -386,14 +395,16 @@ public class CacheExporter @Inject constructor(
FROM sources s FROM sources s
JOIN game_variants v ON v.id = s.game_id JOIN game_variants v ON v.id = s.game_id
JOIN games g ON g.id = v.game_id JOIN games g ON g.id = v.game_id
JOIN scopes sc ON sc.id = g.scope_id
JOIN environments e ON e.id = v.environment_id JOIN environments e ON e.id = v.environment_id
JOIN languages l ON l.id = v.language_id JOIN languages l ON l.id = v.language_id
WHERE s.cache_id = ? WHERE sc.name = ? AND s.cache_id = ?
GROUP BY g.name, e.name, l.iso_code GROUP BY g.name, e.name, l.iso_code
LIMIT 1 LIMIT 1
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, id) stmt.setString(1, scope)
stmt.setInt(2, id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
if (!rows.next()) { if (!rows.next()) {
@ -431,7 +442,7 @@ public class CacheExporter @Inject constructor(
} }
} }
public fun export(id: Int, storeFactory: (Boolean) -> Store) { public fun export(scope: String, id: Int, storeFactory: (Boolean) -> Store) {
database.executeOnce { connection -> database.executeOnce { connection ->
val legacy = connection.prepareStatement( val legacy = connection.prepareStatement(
""" """
@ -451,22 +462,24 @@ public class CacheExporter @Inject constructor(
if (legacy) { if (legacy) {
exportLegacy(connection, id, store) exportLegacy(connection, id, store)
} else { } else {
export(connection, id, store) export(connection, scope, id, store)
} }
} }
} }
} }
private fun export(connection: Connection, id: Int, store: Store) { private fun export(connection: Connection, scope: String, id: Int, store: Store) {
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT archive_id, group_id, data, version SELECT g.archive_id, g.group_id, g.data, g.version
FROM resolved_groups FROM resolved_groups g
WHERE master_index_id = ? JOIN scopes s ON s.id = g.scope_id
WHERE s.name = ? AND g.master_index_id = ?
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.fetchSize = BATCH_SIZE stmt.fetchSize = BATCH_SIZE
stmt.setInt(1, id) stmt.setString(1, scope)
stmt.setInt(2, id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
alloc.buffer(2, 2).use { versionBuf -> alloc.buffer(2, 2).use { versionBuf ->
@ -534,18 +547,20 @@ public class CacheExporter @Inject constructor(
} }
} }
public suspend fun exportKeys(id: Int): List<Key> { public suspend fun exportKeys(scope: String, id: Int): List<Key> {
return database.execute { connection -> return database.execute { connection ->
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT g.archive_id, g.group_id, g.name_hash, n.name, (k.key).k0, (k.key).k1, (k.key).k2, (k.key).k3 SELECT g.archive_id, g.group_id, g.name_hash, n.name, (k.key).k0, (k.key).k1, (k.key).k2, (k.key).k3
FROM resolved_groups g FROM resolved_groups g
JOIN scopes s ON s.id = g.scope_id
JOIN keys k ON k.id = g.key_id JOIN keys k ON k.id = g.key_id
LEFT JOIN names n ON n.hash = g.name_hash AND n.name ~ '^l(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' LEFT JOIN names n ON n.hash = g.name_hash AND n.name ~ '^l(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'
WHERE g.master_index_id = ? WHERE s.name = ? AND g.master_index_id = ?
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, id) stmt.setString(1, scope)
stmt.setInt(2, id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
val keys = mutableListOf<Key>() val keys = mutableListOf<Key>()

@ -116,9 +116,14 @@ public class CacheImporter @Inject constructor(
val indexes: List<ByteBuf?> val indexes: List<ByteBuf?>
) )
private data class Game(
val id: Int,
val scopeId: Int
)
public suspend fun import( public suspend fun import(
store: Store, store: Store,
game: String, gameName: String,
environment: String, environment: String,
language: String, language: String,
buildMajor: Int?, buildMajor: Int?,
@ -131,12 +136,12 @@ public class CacheImporter @Inject constructor(
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
val gameId = getGameId(connection, game, environment, language) val game = getGame(connection, gameName, environment, language)
if (store is DiskStore && store.legacy) { if (store is DiskStore && store.legacy) {
importLegacy(connection, store, gameId, buildMajor, buildMinor, timestamp, name, description, url) importLegacy(connection, store, game.id, buildMajor, buildMinor, timestamp, name, description, url)
} else { } else {
importJs5(connection, store, gameId, buildMajor, buildMinor, timestamp, name, description, url) importJs5(connection, store, game, buildMajor, buildMinor, timestamp, name, description, url)
} }
} }
} }
@ -144,7 +149,7 @@ public class CacheImporter @Inject constructor(
private fun importJs5( private fun importJs5(
connection: Connection, connection: Connection,
store: Store, store: Store,
gameId: Int, game: Game,
buildMajor: Int?, buildMajor: Int?,
buildMinor: Int?, buildMinor: Int?,
timestamp: Instant?, timestamp: Instant?,
@ -169,7 +174,7 @@ public class CacheImporter @Inject constructor(
connection, connection,
SourceType.DISK, SourceType.DISK,
masterIndexId, masterIndexId,
gameId, game.id,
buildMajor, buildMajor,
buildMinor, buildMinor,
timestamp, timestamp,
@ -194,7 +199,7 @@ public class CacheImporter @Inject constructor(
} }
for (index in indexGroups) { for (index in indexGroups) {
addIndex(connection, sourceId, index) addIndex(connection, game.scopeId, sourceId, index)
} }
} finally { } finally {
indexGroups.forEach(Index::release) indexGroups.forEach(Index::release)
@ -215,7 +220,7 @@ public class CacheImporter @Inject constructor(
groups += group groups += group
if (groups.size >= BATCH_SIZE) { if (groups.size >= BATCH_SIZE) {
addGroups(connection, sourceId, groups) addGroups(connection, game.scopeId, sourceId, groups)
groups.forEach(Group::release) groups.forEach(Group::release)
groups.clear() groups.clear()
@ -224,7 +229,7 @@ public class CacheImporter @Inject constructor(
} }
if (groups.isNotEmpty()) { if (groups.isNotEmpty()) {
addGroups(connection, sourceId, groups) addGroups(connection, game.scopeId, sourceId, groups)
} }
} finally { } finally {
groups.forEach(Group::release) groups.forEach(Group::release)
@ -234,7 +239,7 @@ public class CacheImporter @Inject constructor(
public suspend fun importMasterIndex( public suspend fun importMasterIndex(
buf: ByteBuf, buf: ByteBuf,
format: MasterIndexFormat, format: MasterIndexFormat,
game: String, gameName: String,
environment: String, environment: String,
language: String, language: String,
buildMajor: Int?, buildMajor: Int?,
@ -254,14 +259,14 @@ public class CacheImporter @Inject constructor(
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
val gameId = getGameId(connection, game, environment, language) val game = getGame(connection, gameName, environment, language)
val masterIndexId = addMasterIndex(connection, masterIndex) val masterIndexId = addMasterIndex(connection, masterIndex)
addSource( addSource(
connection, connection,
SourceType.DISK, SourceType.DISK,
masterIndexId, masterIndexId,
gameId, game.id,
buildMajor, buildMajor,
buildMinor, buildMinor,
timestamp, timestamp,
@ -363,6 +368,7 @@ public class CacheImporter @Inject constructor(
} }
public suspend fun importIndexAndGetMissingGroups( public suspend fun importIndexAndGetMissingGroups(
scopeId: Int,
sourceId: Int, sourceId: Int,
archive: Int, archive: Int,
index: Js5Index, index: Js5Index,
@ -372,7 +378,7 @@ public class CacheImporter @Inject constructor(
): List<Int> { ): List<Int> {
return database.execute { connection -> return database.execute { connection ->
prepare(connection) prepare(connection)
val id = addIndex(connection, sourceId, Index(archive, index, buf, uncompressed)) val id = addIndex(connection, scopeId, sourceId, Index(archive, index, buf, uncompressed))
/* /*
* In order to defend against (crc32, version) collisions, we only * In order to defend against (crc32, version) collisions, we only
@ -415,14 +421,14 @@ public class CacheImporter @Inject constructor(
} }
} }
public suspend fun importGroups(sourceId: Int, groups: List<Group>) { public suspend fun importGroups(scopeId: Int, sourceId: Int, groups: List<Group>) {
if (groups.isEmpty()) { if (groups.isEmpty()) {
return return
} }
database.execute { connection -> database.execute { connection ->
prepare(connection) prepare(connection)
addGroups(connection, sourceId, groups) addGroups(connection, scopeId, sourceId, groups)
} }
} }
@ -627,22 +633,23 @@ public class CacheImporter @Inject constructor(
} }
} }
private fun addGroups(connection: Connection, sourceId: Int, groups: List<Group>): List<Long> { private fun addGroups(connection: Connection, scopeId: Int, sourceId: Int, groups: List<Group>): List<Long> {
val containerIds = addContainers(connection, groups) val containerIds = addContainers(connection, groups)
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO groups (archive_id, group_id, version, version_truncated, container_id) INSERT INTO groups (scope_id, archive_id, group_id, version, version_truncated, container_id)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT DO NOTHING ON CONFLICT DO NOTHING
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
for ((i, group) in groups.withIndex()) { for ((i, group) in groups.withIndex()) {
stmt.setInt(1, group.archive) stmt.setInt(1, scopeId)
stmt.setInt(2, group.group) stmt.setInt(2, group.archive)
stmt.setInt(3, group.version) stmt.setInt(3, group.group)
stmt.setBoolean(4, group.versionTruncated) stmt.setInt(4, group.version)
stmt.setLong(5, containerIds[i]) stmt.setBoolean(5, group.versionTruncated)
stmt.setLong(6, containerIds[i])
stmt.addBatch() stmt.addBatch()
} }
@ -672,8 +679,8 @@ public class CacheImporter @Inject constructor(
return containerIds return containerIds
} }
private fun addGroup(connection: Connection, sourceId: Int, group: Group): Long { private fun addGroup(connection: Connection, scopeId: Int, sourceId: Int, group: Group): Long {
return addGroups(connection, sourceId, listOf(group)).single() return addGroups(connection, scopeId, sourceId, listOf(group)).single()
} }
private fun readIndex(store: Store, archive: Int): Index { private fun readIndex(store: Store, archive: Int): Index {
@ -684,8 +691,8 @@ public class CacheImporter @Inject constructor(
} }
} }
private fun addIndex(connection: Connection, sourceId: Int, index: Index): Long { private fun addIndex(connection: Connection, scopeId: Int, sourceId: Int, index: Index): Long {
val containerId = addGroup(connection, sourceId, index) val containerId = addGroup(connection, scopeId, sourceId, index)
val savepoint = connection.setSavepoint() val savepoint = connection.setSavepoint()
connection.prepareStatement( connection.prepareStatement(
@ -964,10 +971,10 @@ public class CacheImporter @Inject constructor(
return ids return ids
} }
private fun getGameId(connection: Connection, name: String, environment: String, language: String): Int { private fun getGame(connection: Connection, name: String, environment: String, language: String): Game {
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT v.id SELECT v.id, g.scope_id
FROM game_variants v FROM game_variants v
JOIN games g ON g.id = v.game_id JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id JOIN environments e ON e.id = v.environment_id
@ -984,7 +991,10 @@ public class CacheImporter @Inject constructor(
throw Exception("Game not found") throw Exception("Game not found")
} }
return rows.getInt(1) val id = rows.getInt(1)
val scopeId = rows.getInt(2)
return Game(id, scopeId)
} }
} }
} }

@ -2,6 +2,8 @@ package org.openrs2.archive.cache
import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.arguments.argument
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.types.int import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path import com.github.ajalt.clikt.parameters.types.path
import com.google.inject.Guice import com.google.inject.Guice
@ -11,6 +13,7 @@ import org.openrs2.cache.DiskStore
import org.openrs2.inject.CloseableInjector import org.openrs2.inject.CloseableInjector
public class ExportCommand : CliktCommand(name = "export") { public class ExportCommand : CliktCommand(name = "export") {
private val scope by option().default("runescape")
private val id by argument().int() private val id by argument().int()
private val output by argument().path( private val output by argument().path(
mustExist = true, mustExist = true,
@ -23,7 +26,7 @@ public class ExportCommand : CliktCommand(name = "export") {
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector ->
val exporter = injector.getInstance(CacheExporter::class.java) val exporter = injector.getInstance(CacheExporter::class.java)
exporter.export(id) { legacy -> exporter.export(scope, id) { legacy ->
DiskStore.create(output, legacy = legacy) DiskStore.create(output, legacy = legacy)
} }
} }

@ -25,6 +25,7 @@ import kotlin.coroutines.resumeWithException
@ChannelHandler.Sharable @ChannelHandler.Sharable
public abstract class Js5ChannelHandler( public abstract class Js5ChannelHandler(
private val bootstrap: Bootstrap, private val bootstrap: Bootstrap,
private val scopeId: Int,
private val gameId: Int, private val gameId: Int,
private val hostname: String, private val hostname: String,
private val port: Int, private val port: Int,
@ -236,7 +237,7 @@ public abstract class Js5ChannelHandler(
if (groups.size >= CacheImporter.BATCH_SIZE || complete) { if (groups.size >= CacheImporter.BATCH_SIZE || complete) {
runBlocking { runBlocking {
importer.importGroups(sourceId, groups) importer.importGroups(sourceId, scopeId, groups)
} }
releaseGroups() releaseGroups()
@ -315,7 +316,15 @@ public abstract class Js5ChannelHandler(
} }
val groups = runBlocking { val groups = runBlocking {
importer.importIndexAndGetMissingGroups(sourceId, archive, index, buf, uncompressed, lastMasterIndexId) importer.importIndexAndGetMissingGroups(
scopeId,
sourceId,
archive,
index,
buf,
uncompressed,
lastMasterIndexId
)
} }
for (group in groups) { for (group in groups) {
val groupEntry = index[group]!! val groupEntry = index[group]!!

@ -24,6 +24,7 @@ import kotlin.coroutines.Continuation
public class NxtJs5ChannelHandler( public class NxtJs5ChannelHandler(
bootstrap: Bootstrap, bootstrap: Bootstrap,
scopeId: Int,
gameId: Int, gameId: Int,
hostname: String, hostname: String,
port: Int, port: Int,
@ -38,6 +39,7 @@ public class NxtJs5ChannelHandler(
private val maxMinorBuildAttempts: Int = 5 private val maxMinorBuildAttempts: Int = 5
) : Js5ChannelHandler( ) : Js5ChannelHandler(
bootstrap, bootstrap,
scopeId,
gameId, gameId,
hostname, hostname,
port, port,

@ -17,6 +17,7 @@ import kotlin.coroutines.Continuation
public class OsrsJs5ChannelHandler( public class OsrsJs5ChannelHandler(
bootstrap: Bootstrap, bootstrap: Bootstrap,
scopeId: Int,
gameId: Int, gameId: Int,
hostname: String, hostname: String,
port: Int, port: Int,
@ -26,6 +27,7 @@ public class OsrsJs5ChannelHandler(
importer: CacheImporter, importer: CacheImporter,
) : Js5ChannelHandler( ) : Js5ChannelHandler(
bootstrap, bootstrap,
scopeId,
gameId, gameId,
hostname, hostname,
port, port,

@ -6,5 +6,6 @@ public data class Game(
public val buildMajor: Int?, public val buildMajor: Int?,
public val buildMinor: Int?, public val buildMinor: Int?,
public val lastMasterIndexId: Int?, public val lastMasterIndexId: Int?,
public val languageId: Int public val languageId: Int,
public val scopeId: Int
) )

@ -12,7 +12,7 @@ public class GameDatabase @Inject constructor(
return database.execute { connection -> return database.execute { connection ->
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT v.id, v.url, v.build_major, v.build_minor, v.last_master_index_id, v.language_id SELECT v.id, v.url, v.build_major, v.build_minor, v.last_master_index_id, v.language_id, g.scope_id
FROM game_variants v FROM game_variants v
JOIN games g ON g.id = v.game_id JOIN games g ON g.id = v.game_id
JOIN environments e ON e.id = v.environment_id JOIN environments e ON e.id = v.environment_id
@ -48,8 +48,9 @@ public class GameDatabase @Inject constructor(
} }
val languageId = rows.getInt(6) val languageId = rows.getInt(6)
val scopeId = rows.getInt(7)
return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId, languageId) return@execute Game(id, url, buildMajor, buildMinor, lastMasterIndexId, languageId, scopeId)
} }
} }
} }

@ -31,10 +31,28 @@ public class MapRenderer @Inject constructor(
val fillColor = Color(outlineColor.red, outlineColor.green, outlineColor.blue, 128) val fillColor = Color(outlineColor.red, outlineColor.green, outlineColor.blue, 128)
} }
public suspend fun render(masterIndexId: Int): BufferedImage { public suspend fun render(scope: String, masterIndexId: Int): BufferedImage {
return database.execute { connection -> return database.execute { connection ->
val scopeId = connection.prepareStatement(
"""
SELECT id
FROM scopes
WHERE name = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, scope)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
throw IllegalArgumentException("Invalid scope")
}
rows.getInt(1)
}
}
// read config index // read config index
val configIndex = readIndex(connection, masterIndexId, Js5Archive.CONFIG) val configIndex = readIndex(connection, scopeId, masterIndexId, Js5Archive.CONFIG)
?: throw IllegalArgumentException("Config index missing") ?: throw IllegalArgumentException("Config index missing")
// read FluType group // read FluType group
@ -43,7 +61,7 @@ public class MapRenderer @Inject constructor(
val underlayGroup = configIndex[Js5ConfigGroup.FLUTYPE] val underlayGroup = configIndex[Js5ConfigGroup.FLUTYPE]
?: throw IllegalArgumentException("FluType group missing in index") ?: throw IllegalArgumentException("FluType group missing in index")
val underlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, underlayGroup) val underlayFiles = readGroup(connection, scopeId, masterIndexId, Js5Archive.CONFIG, underlayGroup)
?: throw IllegalArgumentException("FluType group missing") ?: throw IllegalArgumentException("FluType group missing")
try { try {
for ((id, file) in underlayFiles) { for ((id, file) in underlayFiles) {
@ -59,7 +77,7 @@ public class MapRenderer @Inject constructor(
val overlayGroup = configIndex[Js5ConfigGroup.FLOTYPE] val overlayGroup = configIndex[Js5ConfigGroup.FLOTYPE]
?: throw IllegalArgumentException("FloType group missing in index") ?: throw IllegalArgumentException("FloType group missing in index")
val overlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, overlayGroup) val overlayFiles = readGroup(connection, scopeId, masterIndexId, Js5Archive.CONFIG, overlayGroup)
?: throw IllegalArgumentException("FloType group missing") ?: throw IllegalArgumentException("FloType group missing")
try { try {
for ((id, file) in overlayFiles) { for ((id, file) in overlayFiles) {
@ -71,13 +89,13 @@ public class MapRenderer @Inject constructor(
// read textures // read textures
val textures = mutableMapOf<Int, Int>() val textures = mutableMapOf<Int, Int>()
val materialsIndex = readIndex(connection, masterIndexId, Js5Archive.MATERIALS) val materialsIndex = readIndex(connection, scopeId, masterIndexId, Js5Archive.MATERIALS)
if (materialsIndex != null) { if (materialsIndex != null) {
val materialsGroup = materialsIndex[0] val materialsGroup = materialsIndex[0]
?: throw IllegalArgumentException("Materials group missing in index") ?: throw IllegalArgumentException("Materials group missing in index")
val materialsFiles = readGroup(connection, masterIndexId, Js5Archive.MATERIALS, materialsGroup) val materialsFiles = readGroup(connection, scopeId, masterIndexId, Js5Archive.MATERIALS, materialsGroup)
?: throw IllegalArgumentException("Materials group missing") ?: throw IllegalArgumentException("Materials group missing")
try { try {
val metadata = materialsFiles[0] val metadata = materialsFiles[0]
@ -123,13 +141,13 @@ public class MapRenderer @Inject constructor(
materialsFiles.values.forEach(ByteBuf::release) materialsFiles.values.forEach(ByteBuf::release)
} }
} else { } else {
val textureIndex = readIndex(connection, masterIndexId, Js5Archive.TEXTURES) val textureIndex = readIndex(connection, scopeId, masterIndexId, Js5Archive.TEXTURES)
?: throw IllegalArgumentException("Textures index missing") ?: throw IllegalArgumentException("Textures index missing")
val textureGroup = textureIndex[0] val textureGroup = textureIndex[0]
?: throw IllegalArgumentException("Textures group missing from index") ?: throw IllegalArgumentException("Textures group missing from index")
val textureFiles = readGroup(connection, masterIndexId, Js5Archive.TEXTURES, textureGroup) val textureFiles = readGroup(connection, scopeId, masterIndexId, Js5Archive.TEXTURES, textureGroup)
?: throw IllegalArgumentException("Textures group missing") ?: throw IllegalArgumentException("Textures group missing")
try { try {
for ((id, file) in textureFiles) { for ((id, file) in textureFiles) {
@ -155,11 +173,12 @@ public class MapRenderer @Inject constructor(
SELECT n.name, g.encrypted, g.empty_loc, g.key_id SELECT n.name, g.encrypted, g.empty_loc, g.key_id
FROM resolved_groups g FROM resolved_groups g
JOIN names n ON n.hash = g.name_hash JOIN names n ON n.hash = g.name_hash
WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND WHERE g.scope_id = ? AND g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND
n.name ~ '^[lm](?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' n.name ~ '^[lm](?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, masterIndexId) stmt.setInt(1, scopeId)
stmt.setInt(2, masterIndexId)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
while (rows.next()) { while (rows.next()) {
@ -207,11 +226,12 @@ public class MapRenderer @Inject constructor(
SELECT n.name, g.data SELECT n.name, g.data
FROM resolved_groups g FROM resolved_groups g
JOIN names n ON n.hash = g.name_hash JOIN names n ON n.hash = g.name_hash
WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND WHERE g.scope_id = ? AND g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND
n.name ~ '^m(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' n.name ~ '^m(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, masterIndexId) stmt.setInt(1, scopeId)
stmt.setInt(2, masterIndexId)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
while (rows.next()) { while (rows.next()) {
@ -246,16 +266,17 @@ public class MapRenderer @Inject constructor(
} }
} }
private fun readIndex(connection: Connection, masterIndexId: Int, archiveId: Int): Js5Index? { private fun readIndex(connection: Connection, scopeId: Int, masterIndexId: Int, archiveId: Int): Js5Index? {
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT data SELECT data
FROM resolved_indexes FROM resolved_indexes
WHERE master_index_id = ? AND archive_id = ? WHERE scope_id = ? AND master_index_id = ? AND archive_id = ?
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, masterIndexId) stmt.setInt(1, scopeId)
stmt.setInt(2, archiveId) stmt.setInt(2, masterIndexId)
stmt.setInt(3, archiveId)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
if (!rows.next()) { if (!rows.next()) {
@ -275,6 +296,7 @@ public class MapRenderer @Inject constructor(
private fun readGroup( private fun readGroup(
connection: Connection, connection: Connection,
scopeId: Int,
masterIndexId: Int, masterIndexId: Int,
archiveId: Int, archiveId: Int,
group: Js5Index.Group<*> group: Js5Index.Group<*>
@ -283,12 +305,13 @@ public class MapRenderer @Inject constructor(
""" """
SELECT data SELECT data
FROM resolved_groups FROM resolved_groups
WHERE master_index_id = ? AND archive_id = ? AND group_id = ? WHERE scope_id = ? AND master_index_id = ? AND archive_id = ? AND group_id = ?
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.setInt(1, masterIndexId) stmt.setInt(1, scopeId)
stmt.setInt(2, archiveId) stmt.setInt(2, masterIndexId)
stmt.setInt(3, group.id) stmt.setInt(3, archiveId)
stmt.setInt(4, group.id)
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
if (!rows.next()) { if (!rows.next()) {

@ -46,29 +46,38 @@ public class CachesController @Inject constructor(
} }
public suspend fun show(call: ApplicationCall) { public suspend fun show(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val cache = exporter.get(id) val cache = exporter.get(scope, id)
if (cache == null) { if (cache == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
call.respond(ThymeleafContent("caches/show.html", mapOf("cache" to cache))) call.respond(
ThymeleafContent(
"caches/show.html", mapOf(
"cache" to cache,
"scope" to scope,
)
)
)
} }
public suspend fun exportDisk(call: ApplicationCall) { public suspend fun exportDisk(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val name = exporter.getFileName(id) val name = exporter.getFileName(scope, id)
if (name == null) { if (name == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
@ -82,20 +91,21 @@ public class CachesController @Inject constructor(
) )
call.respondOutputStream(contentType = ContentType.Application.Zip) { call.respondOutputStream(contentType = ContentType.Application.Zip) {
exporter.export(id) { legacy -> exporter.export(scope, id) { legacy ->
DiskStoreZipWriter(ZipOutputStream(this), alloc = alloc, legacy = legacy) DiskStoreZipWriter(ZipOutputStream(this), alloc = alloc, legacy = legacy)
} }
} }
} }
public suspend fun exportFlatFile(call: ApplicationCall) { public suspend fun exportFlatFile(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val name = exporter.getFileName(id) val name = exporter.getFileName(scope, id)
if (name == null) { if (name == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
@ -109,20 +119,21 @@ public class CachesController @Inject constructor(
) )
call.respondOutputStream(contentType = ContentType.Application.GZip) { call.respondOutputStream(contentType = ContentType.Application.GZip) {
exporter.export(id) { exporter.export(scope, id) {
FlatFileStoreTarWriter(TarArchiveOutputStream(GzipLevelOutputStream(this, Deflater.BEST_COMPRESSION))) FlatFileStoreTarWriter(TarArchiveOutputStream(GzipLevelOutputStream(this, Deflater.BEST_COMPRESSION)))
} }
} }
} }
public suspend fun exportKeysJson(call: ApplicationCall) { public suspend fun exportKeysJson(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val name = exporter.getFileName(id) val name = exporter.getFileName(scope, id)
if (name == null) { if (name == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
@ -135,17 +146,18 @@ public class CachesController @Inject constructor(
.toString() .toString()
) )
call.respond(exporter.exportKeys(id)) call.respond(exporter.exportKeys(scope, id))
} }
public suspend fun exportKeysZip(call: ApplicationCall) { public suspend fun exportKeysZip(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val name = exporter.getFileName(id) val name = exporter.getFileName(scope, id)
if (name == null) { if (name == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
@ -165,7 +177,7 @@ public class CachesController @Inject constructor(
val timestamp = FileTime.from(Instant.EPOCH) val timestamp = FileTime.from(Instant.EPOCH)
for (key in exporter.exportKeys(id)) { for (key in exporter.exportKeys(scope, id)) {
if (key.mapSquare == null) { if (key.mapSquare == null) {
continue continue
} }
@ -197,13 +209,14 @@ public class CachesController @Inject constructor(
} }
public suspend fun renderMap(call: ApplicationCall) { public suspend fun renderMap(call: ApplicationCall) {
val scope = call.parameters["scope"]!!
val id = call.parameters["id"]?.toIntOrNull() val id = call.parameters["id"]?.toIntOrNull()
if (id == null) { if (id == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
} }
val name = exporter.getFileName(id) val name = exporter.getFileName(scope, id)
if (name == null) { if (name == null) {
call.respond(HttpStatusCode.NotFound) call.respond(HttpStatusCode.NotFound)
return return
@ -222,7 +235,7 @@ public class CachesController @Inject constructor(
* performed in parallel to prevent OOMs. * performed in parallel to prevent OOMs.
*/ */
renderSemaphore.withPermit { renderSemaphore.withPermit {
val image = renderer.render(id) val image = renderer.render(scope, id)
call.respondOutputStream(contentType = ContentType.Image.PNG) { call.respondOutputStream(contentType = ContentType.Image.PNG) {
ImageIO.write(image, "PNG", this) ImageIO.write(image, "PNG", this)

@ -1,12 +1,12 @@
package org.openrs2.archive.web package org.openrs2.archive.web
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import io.ktor.application.ApplicationCall
import io.ktor.application.call import io.ktor.application.call
import io.ktor.application.install import io.ktor.application.install
import io.ktor.features.ContentNegotiation import io.ktor.features.ContentNegotiation
import io.ktor.features.XForwardedHeaderSupport import io.ktor.features.XForwardedHeaderSupport
import io.ktor.http.ContentType import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.resources import io.ktor.http.content.resources
import io.ktor.http.content.static import io.ktor.http.content.static
import io.ktor.jackson.JacksonConverter import io.ktor.jackson.JacksonConverter
@ -56,40 +56,43 @@ public class WebServer @Inject constructor(
get("/") { call.respond(ThymeleafContent("index.html", emptyMap())) } get("/") { call.respond(ThymeleafContent("index.html", emptyMap())) }
get("/caches") { cachesController.index(call) } get("/caches") { cachesController.index(call) }
get("/caches.json") { cachesController.indexJson(call) } get("/caches.json") { cachesController.indexJson(call) }
get("/caches/{id}") { cachesController.show(call) } get("/caches/{scope}/{id}") { cachesController.show(call) }
get("/caches/{id}.zip") { get("/caches/{scope}/{id}/disk.zip") { cachesController.exportDisk(call) }
val id = call.parameters["id"] get("/caches/{scope}/{id}/flat-file.tar.gz") { cachesController.exportFlatFile(call) }
if (id == null) { get("/caches/{scope}/{id}/keys.json") { cachesController.exportKeysJson(call) }
call.respond(HttpStatusCode.NotFound) get("/caches/{scope}/{id}/keys.zip") { cachesController.exportKeysZip(call) }
return@get get("/caches/{scope}/{id}/map.png") { cachesController.renderMap(call) }
}
call.respondRedirect(permanent = true) {
path("caches", id, "disk.zip")
}
}
get("/caches/{id}.json") {
val id = call.parameters["id"]
if (id == null) {
call.respond(HttpStatusCode.NotFound)
return@get
}
call.respondRedirect(permanent = true) {
path("caches", id, "keys.json")
}
}
get("/caches/{id}/disk.zip") { cachesController.exportDisk(call) }
get("/caches/{id}/flat-file.tar.gz") { cachesController.exportFlatFile(call) }
get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) }
get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) }
get("/caches/{id}/map.png") { cachesController.renderMap(call) }
get("/keys") { keysController.index(call) } get("/keys") { keysController.index(call) }
post("/keys") { keysController.import(call) } post("/keys") { keysController.import(call) }
get("/keys/all.json") { keysController.exportAll(call) } get("/keys/all.json") { keysController.exportAll(call) }
get("/keys/valid.json") { keysController.exportValid(call) } get("/keys/valid.json") { keysController.exportValid(call) }
static("/static") { resources("/org/openrs2/archive/static") } static("/static") { resources("/org/openrs2/archive/static") }
// compatibility redirects
get("/caches/{id}") { redirect(call, permanent = true, "/caches/runescape/{id}") }
get("/caches/{id}.json") { redirect(call, permanent = true, "/caches/runescape/{id}/keys.json") }
get("/caches/{id}.zip") { redirect(call, permanent = true, "/caches/runescape/{id}/disk.zip") }
get("/caches/{id}/disk.zip") { redirect(call, permanent = true, "/caches/runescape/{id}/disk.zip") }
get("/caches/{id}/flat-file.tar.gz") {
redirect(call, permanent = true, "/caches/runescape/{id}/flat-file.tar.gz")
}
get("/caches/{id}/keys.json") { redirect(call, permanent = true, "/caches/runescape/{id}/keys.json") }
get("/caches/{id}/keys.zip") { redirect(call, permanent = true, "/caches/runescape/{id}/keys.zip") }
get("/caches/{id}/map.png") { redirect(call, permanent = true, "/caches/runescape/{id}/map.png") }
} }
}.start(wait = true) }.start(wait = true)
} }
private suspend fun redirect(call: ApplicationCall, permanent: Boolean, path: String) {
val destination = path.replace(PARAMETER) { match ->
val (name) = match.destructured
call.parameters[name] ?: throw IllegalArgumentException()
}
call.respondRedirect(destination, permanent)
}
private companion object {
private val PARAMETER = Regex("\\{([^}]*)}")
}
} }

@ -0,0 +1,176 @@
-- @formatter:off
CREATE TABLE scopes (
id SERIAL PRIMARY KEY NOT NULL,
name TEXT UNIQUE NOT NULL
);
INSERT INTO scopes (name) VALUES ('runescape');
ALTER TABLE games
ADD COLUMN scope_id INTEGER DEFAULT 1 NOT NULL REFERENCES scopes (id);
ALTER TABLE games
ALTER COLUMN scope_id DROP DEFAULT;
-- XXX(gpe): I don't think we can easily replace this as the source_groups
-- table doesn't contain a scope_id directly - only indirectly via the sources
-- and games tables.
ALTER TABLE source_groups
DROP CONSTRAINT source_groups_archive_id_group_id_version_version_truncate_fkey;
ALTER TABLE groups
ADD COLUMN scope_id INTEGER DEFAULT 1 NOT NULL REFERENCES scopes (id),
DROP CONSTRAINT groups_pkey,
ADD PRIMARY KEY (scope_id, archive_id, group_id, version, version_truncated, container_id);
ALTER TABLE groups
ALTER COLUMN scope_id DROP DEFAULT;
CREATE FUNCTION resolve_index(_scope_id INTEGER, _archive_id uint1, _crc32 INTEGER, _version INTEGER) RETURNS SETOF containers AS $$
SELECT c.*
FROM groups g
JOIN containers c ON c.id = g.container_id
JOIN indexes i ON i.container_id = c.id
WHERE g.scope_id = _scope_id AND g.archive_id = 255 AND g.group_id = _archive_id::INTEGER AND c.crc32 = _crc32 AND
g.version = _version AND NOT g.version_truncated AND i.version = _version
ORDER BY c.id ASC
LIMIT 1;
$$ LANGUAGE SQL STABLE PARALLEL SAFE ROWS 1;
CREATE FUNCTION resolve_group(_scope_id INTEGER, _archive_id uint1, _group_id INTEGER, _crc32 INTEGER, _version INTEGER) RETURNS SETOF containers AS $$
SELECT c.*
FROM groups g
JOIN containers c ON c.id = g.container_id
WHERE g.scope_id = _scope_id AND g.archive_id = _archive_id AND g.group_id = _group_id AND c.crc32 = _crc32 AND (
(g.version = _version AND NOT g.version_truncated) OR
(g.version = _version & 65535 AND g.version_truncated)
)
ORDER BY g.version_truncated ASC, c.id ASC
LIMIT 1;
$$ LANGUAGE SQL STABLE PARALLEL SAFE ROWS 1;
DROP VIEW resolved_groups;
DROP VIEW resolved_indexes;
CREATE VIEW resolved_indexes AS
SELECT s.id AS scope_id, m.id AS master_index_id, a.archive_id, c.data, c.id AS container_id
FROM scopes s
CROSS JOIN master_indexes m
JOIN master_index_archives a ON a.master_index_id = m.id
JOIN resolve_index(s.id, a.archive_id, a.crc32, a.version) c ON TRUE;
CREATE VIEW resolved_groups (scope_id, master_index_id, archive_id, group_id, name_hash, version, data, encrypted, empty_loc, key_id) AS
WITH i AS NOT MATERIALIZED (
SELECT scope_id, master_index_id, archive_id, data, container_id
FROM resolved_indexes
)
SELECT i.scope_id, i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, FALSE, FALSE, NULL
FROM i
UNION ALL
SELECT i.scope_id, i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.encrypted, c.empty_loc, c.key_id
FROM i
JOIN index_groups ig ON ig.container_id = i.container_id
JOIN resolve_group(i.scope_id, i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE;
DROP VIEW colliding_groups;
CREATE VIEW colliding_groups (scope_id, archive_id, group_id, crc32, truncated_version, versions, containers) AS
SELECT
g.scope_id,
g.archive_id,
g.group_id,
c.crc32,
g.version & 65535 AS truncated_version,
array_agg(DISTINCT g.version ORDER BY g.version ASC),
array_agg(DISTINCT c.id ORDER BY c.id ASC)
FROM groups g
JOIN containers c ON c.id = g.container_id
GROUP BY g.scope_id, g.archive_id, g.group_id, c.crc32, truncated_version
HAVING COUNT(DISTINCT c.id) > 1;
DROP VIEW cache_stats;
DROP MATERIALIZED VIEW master_index_stats;
DROP MATERIALIZED VIEW index_stats;
CREATE MATERIALIZED VIEW index_stats (
scope_id,
archive_id,
container_id,
valid_groups,
groups,
valid_keys,
keys,
size,
blocks
) AS
SELECT
s.id AS scope_id,
g.group_id AS archive_id,
i.container_id,
COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups,
COUNT(*) AS groups,
COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys,
COUNT(*) FILTER (WHERE c.encrypted) AS keys,
SUM(length(c.data) + 2) FILTER (WHERE c.id IS NOT NULL) AS size,
SUM(group_blocks(ig.group_id, length(c.data) + 2)) FILTER (WHERE c.id IS NOT NULL) AS blocks
FROM scopes s
CROSS JOIN indexes i
JOIN groups g ON g.container_id = i.container_id AND g.archive_id = 255 AND NOT g.version_truncated AND
g.version = i.version
JOIN index_groups ig ON ig.container_id = i.container_id
LEFT JOIN resolve_group(s.id, g.group_id::uint1, ig.group_id, ig.crc32, ig.version) c ON TRUE
GROUP BY s.id, g.group_id, i.container_id;
CREATE UNIQUE INDEX ON index_stats (scope_id, archive_id, container_id);
CREATE MATERIALIZED VIEW master_index_stats (
scope_id,
master_index_id,
valid_indexes,
indexes,
valid_groups,
groups,
valid_keys,
keys,
size,
blocks
) AS
SELECT
sc.id,
m.id,
COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes,
COUNT(*) FILTER (WHERE a.master_index_id IS NOT NULL) AS indexes,
SUM(COALESCE(s.valid_groups, 0)) AS valid_groups,
SUM(COALESCE(s.groups, 0)) AS groups,
SUM(COALESCE(s.valid_keys, 0)) AS valid_keys,
SUM(COALESCE(s.keys, 0)) AS keys,
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(c.data), 0)) AS size,
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(c.data)), 0)) AS blocks
FROM scopes sc
CROSS JOIN master_indexes m
LEFT JOIN master_index_archives a ON a.master_index_id = m.id
LEFT JOIN resolve_index(sc.id, a.archive_id, a.crc32, a.version) c ON TRUE
LEFT JOIN index_stats s ON s.scope_id = sc.id AND s.archive_id = a.archive_id AND s.container_id = c.id
GROUP BY sc.id, m.id;
CREATE UNIQUE INDEX ON master_index_stats (scope_id, master_index_id);
CREATE VIEW cache_stats AS
SELECT
s.id AS scope_id,
c.id AS cache_id,
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes,
COALESCE(ms.indexes, cs.archives) AS indexes,
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups,
COALESCE(ms.groups, cs.files) AS groups,
COALESCE(ms.valid_keys, 0) AS valid_keys,
COALESCE(ms.keys, 0) AS keys,
COALESCE(ms.size, cs.size) AS size,
COALESCE(ms.blocks, cs.blocks) AS blocks
FROM scopes s
CROSS JOIN caches c
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id;
DROP FUNCTION resolve_group(_archive_id uint1, _group_id INTEGER, _crc32 INTEGER, _version INTEGER);
DROP FUNCTION resolve_index(_archive_id uint1, _crc32 INTEGER, _version INTEGER);

@ -84,25 +84,25 @@
</button> </button>
<ul class="dropdown-menu"> <ul class="dropdown-menu">
<li th:if="${cache.stats != null and cache.stats.diskStoreValid}"><a <li th:if="${cache.stats != null and cache.stats.diskStoreValid}"><a
th:href="${'/caches/' + cache.id + '/disk.zip'}" th:href="${'/caches/' + cache.scope + '/' + cache.id + '/disk.zip'}"
class="dropdown-item">Cache (.dat2/.idx)</a></li> class="dropdown-item">Cache (.dat2/.idx)</a></li>
<li><a th:href="${'/caches/' + cache.id + '/flat-file.tar.gz'}" <li><a th:href="${'/caches/' + cache.scope + '/' + cache.id + '/flat-file.tar.gz'}"
class="dropdown-item">Cache (Flat file)</a></li> class="dropdown-item">Cache (Flat file)</a></li>
<li> <li>
<hr class="dropdown-divider" /> <hr class="dropdown-divider" />
</li> </li>
<li><a th:href="${'/caches/' + cache.id + '/keys.json'}" <li><a th:href="${'/caches/' + cache.scope + '/' + cache.id + '/keys.json'}"
class="dropdown-item">Keys (JSON)</a></li> class="dropdown-item">Keys (JSON)</a></li>
<li><a th:href="${'/caches/' + cache.id + '/keys.zip'}" <li><a th:href="${'/caches/' + cache.scope + '/' + cache.id + '/keys.zip'}"
class="dropdown-item">Keys (Text)</a></li> class="dropdown-item">Keys (Text)</a></li>
<li> <li>
<hr class="dropdown-divider" /> <hr class="dropdown-divider" />
</li> </li>
<li><a th:href="${'/caches/' + cache.id + '/map.png'}" <li><a th:href="${'/caches/' + cache.scope + '/' + cache.id + '/map.png'}"
class="dropdown-item">Map</a></li> class="dropdown-item">Map</a></li>
</ul> </ul>
</div> </div>
<a th:href="${'/caches/' + cache.id}" <a th:href="${'/caches/' + cache.scope + '/' + cache.id}"
class="btn btn-secondary btn-sm">More</a> class="btn btn-secondary btn-sm">More</a>
</div> </div>
</td> </td>

@ -52,19 +52,19 @@
<div class="btn-toolbar"> <div class="btn-toolbar">
<div class="btn-group me-2"> <div class="btn-group me-2">
<a th:if="${cache.stats != null and cache.stats.diskStoreValid}" <a th:if="${cache.stats != null and cache.stats.diskStoreValid}"
th:href="${'/caches/' + cache.id + '/disk.zip'}" th:href="${'/caches/' + scope + '/' + cache.id + '/disk.zip'}"
class="btn btn-primary btn-sm">Cache (.dat2/.idx)</a> class="btn btn-primary btn-sm">Cache (.dat2/.idx)</a>
<a th:href="${'/caches/' + cache.id + '/flat-file.tar.gz'}" <a th:href="${'/caches/' + scope + '/' + cache.id + '/flat-file.tar.gz'}"
class="btn btn-primary btn-sm">Cache (Flat file)</a> class="btn btn-primary btn-sm">Cache (Flat file)</a>
</div> </div>
<div class="btn-group me-2"> <div class="btn-group me-2">
<a th:href="${'/caches/' + cache.id + '/keys.json'}" <a th:href="${'/caches/' + scope + '/' + cache.id + '/keys.json'}"
class="btn btn-primary btn-sm">Keys (JSON)</a> class="btn btn-primary btn-sm">Keys (JSON)</a>
<a th:href="${'/caches/' + cache.id + '/keys.zip'}" <a th:href="${'/caches/' + scope + '/' + cache.id + '/keys.zip'}"
class="btn btn-primary btn-sm">Keys (Text)</a> class="btn btn-primary btn-sm">Keys (Text)</a>
</div> </div>
<div class="btn-group"> <div class="btn-group">
<a th:href="${'/caches/' + cache.id + '/map.png'}" <a th:href="${'/caches/' + scope + '/' + cache.id + '/map.png'}"
class="btn btn-primary btn-sm">Map</a> class="btn btn-primary btn-sm">Map</a>
</div> </div>
</div> </div>

Loading…
Cancel
Save