diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt index a5f8839a..17bc9f7f 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt @@ -12,9 +12,10 @@ import java.net.http.HttpResponse import java.time.Duration public abstract class JsonKeyDownloader( + source: KeySource, private val client: HttpClient, private val jsonKeyReader: JsonKeyReader -) : KeyDownloader { +) : KeyDownloader(source) { override suspend fun download(url: String): Sequence { val request = HttpRequest.newBuilder(URI(url)) .GET() diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeyBruteForcer.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeyBruteForcer.kt index 2f304485..8578f5b5 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/KeyBruteForcer.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeyBruteForcer.kt @@ -22,6 +22,88 @@ public class KeyBruteForcer @Inject constructor( val uncompressedChecksum: Int ) + /* + * Copy XTEA keys from key_queue to keys. The queue exists so that we don't + * block the /keys API endpoint from working while the brute forcer is + * running. + * + * This has to be a different transaction as it needs to lock the keys + * table in EXCLUSIVE mode, but we want to downgrade that to SHARE mode as + * soon as possible. Locks can only be released on commit in Postgres. + */ + private suspend fun assignKeyIds() { + database.execute { connection -> + connection.prepareStatement( + """ + LOCK TABLE keys IN EXCLUSIVE MODE + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + CREATE TEMPORARY TABLE tmp_keys ( + key xtea_key NOT NULL, + source key_source NOT NULL, + first_seen TIMESTAMPTZ NOT NULL, + last_seen TIMESTAMPTZ NOT NULL + ) ON COMMIT DROP + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO tmp_keys (key, source, first_seen, last_seen) + SELECT key, source, first_seen, last_seen + FROM key_queue + FOR UPDATE SKIP LOCKED + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO keys (key) + SELECT t.key + FROM tmp_keys t + LEFT JOIN keys k ON k.key = t.key + WHERE k.key IS NULL + ON CONFLICT DO NOTHING + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO key_sources AS s (key_id, source, first_seen, last_seen) + SELECT k.id, t.source, t.first_seen, t.last_seen + FROM tmp_keys t + JOIN keys k ON k.key = t.key + ON CONFLICT (key_id, source) DO UPDATE SET + first_seen = LEAST(s.first_seen, EXCLUDED.first_seen), + last_seen = GREATEST(s.last_seen, EXCLUDED.last_seen) + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + DELETE FROM key_queue k + USING tmp_keys t + WHERE k.key = t.key AND k.source = t.source + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + } + } + /* * The code for writing to the containers and keys tables ensures that the * row IDs are allocated monotonically (by forbidding any other @@ -64,6 +146,8 @@ public class KeyBruteForcer @Inject constructor( * from the tables. */ public suspend fun bruteForce() { + assignKeyIds() + database.execute { connection -> connection.prepareStatement( """ diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt index 2de4a478..202f378b 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt @@ -2,7 +2,9 @@ package org.openrs2.archive.key import org.openrs2.crypto.XteaKey -public interface KeyDownloader { - public suspend fun getMissingUrls(seenUrls: Set): Set - public suspend fun download(url: String): Sequence +public abstract class KeyDownloader( + public val source: KeySource +) { + public abstract suspend fun getMissingUrls(seenUrls: Set): Set + public abstract suspend fun download(url: String): Sequence } diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt index 8035d58a..59fe97f8 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt @@ -6,6 +6,9 @@ import org.openrs2.db.Database import java.nio.file.Files import java.nio.file.Path import java.sql.Connection +import java.sql.Types +import java.time.Instant +import java.time.ZoneOffset import javax.inject.Inject import javax.inject.Singleton @@ -15,6 +18,8 @@ public class KeyImporter @Inject constructor( private val jsonKeyReader: JsonKeyReader, private val downloaders: Set ) { + private data class Key(val key: XteaKey, val source: KeySource) + public suspend fun import(path: Path) { val keys = mutableSetOf() @@ -43,10 +48,12 @@ public class KeyImporter @Inject constructor( logger.info { "Importing ${keys.size} keys" } - import(keys) + import(keys, KeySource.DISK) } public suspend fun download() { + val now = Instant.now() + val seenUrls = database.execute { connection -> connection.prepareStatement( """ @@ -63,12 +70,14 @@ public class KeyImporter @Inject constructor( } } - val keys = mutableSetOf() + val keys = mutableSetOf() val urls = mutableSetOf() for (downloader in downloaders) { for (url in downloader.getMissingUrls(seenUrls)) { - keys += downloader.download(url) + keys += downloader.download(url).map { key -> + Key(key, downloader.source) + } urls += url } } @@ -89,68 +98,49 @@ public class KeyImporter @Inject constructor( stmt.executeBatch() } - import(connection, keys) + import(connection, keys, now) } } - public suspend fun import(keys: Iterable) { + public suspend fun import(keys: Iterable, source: KeySource) { + val now = Instant.now() + database.execute { connection -> - import(connection, keys) + import(connection, keys.map { key -> + Key(key, source) + }, now) } } - private fun import(connection: Connection, keys: Iterable) { - connection.prepareStatement( - """ - LOCK TABLE keys IN EXCLUSIVE MODE - """.trimIndent() - ).use { stmt -> - stmt.execute() - } - - connection.prepareStatement( - """ - CREATE TEMPORARY TABLE tmp_keys ( - key xtea_key PRIMARY KEY NOT NULL - ) ON COMMIT DROP - """.trimIndent() - ).use { stmt -> - stmt.execute() - } + private fun import(connection: Connection, keys: Iterable, now: Instant) { + val timestamp = now.atOffset(ZoneOffset.UTC) connection.prepareStatement( """ - INSERT INTO tmp_keys (key) - VALUES (ROW(?, ?, ?, ?)) + INSERT INTO key_queue AS K (key, source, first_seen, last_seen) + VALUES (ROW(?, ?, ?, ?), ?::key_source, ?, ?) + ON CONFLICT (key, source) DO UPDATE SET + first_seen = LEAST(k.first_seen, EXCLUDED.first_seen), + last_seen = GREATEST(k.last_seen, EXCLUDED.last_seen) """.trimIndent() ).use { stmt -> for (key in keys) { - if (key.isZero) { + if (key.key.isZero) { continue } - stmt.setInt(1, key.k0) - stmt.setInt(2, key.k1) - stmt.setInt(3, key.k2) - stmt.setInt(4, key.k3) + stmt.setInt(1, key.key.k0) + stmt.setInt(2, key.key.k1) + stmt.setInt(3, key.key.k2) + stmt.setInt(4, key.key.k3) + stmt.setString(5, key.source.name.lowercase()) + stmt.setObject(6, timestamp, Types.TIMESTAMP_WITH_TIMEZONE) + stmt.setObject(7, timestamp, Types.TIMESTAMP_WITH_TIMEZONE) stmt.addBatch() } stmt.executeBatch() } - - connection.prepareStatement( - """ - INSERT INTO keys (key) - SELECT t.key - FROM tmp_keys t - LEFT JOIN keys k ON k.key = t.key - WHERE k.key IS NULL - ON CONFLICT DO NOTHING - """.trimIndent() - ).use { stmt -> - stmt.execute() - } } private companion object { diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeySource.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeySource.kt new file mode 100644 index 00000000..878dd214 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeySource.kt @@ -0,0 +1,9 @@ +package org.openrs2.archive.key + +public enum class KeySource { + API, + DISK, + OPENOSRS, + POLAR, + RUNELITE +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt index 8bb88805..8e6cb58f 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt @@ -8,7 +8,7 @@ import javax.inject.Singleton public class OpenOsrsKeyDownloader @Inject constructor( client: HttpClient, jsonKeyReader: JsonKeyReader -) : JsonKeyDownloader(client, jsonKeyReader) { +) : JsonKeyDownloader(KeySource.OPENOSRS, client, jsonKeyReader) { override suspend fun getMissingUrls(seenUrls: Set): Set { return setOf(ENDPOINT) } diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt index 3c7aad48..113ffcb0 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt @@ -18,7 +18,7 @@ import javax.inject.Singleton public class PolarKeyDownloader @Inject constructor( private val client: HttpClient, jsonKeyReader: JsonKeyReader -) : JsonKeyDownloader(client, jsonKeyReader) { +) : JsonKeyDownloader(KeySource.POLAR, client, jsonKeyReader) { override suspend fun getMissingUrls(seenUrls: Set): Set { val request = HttpRequest.newBuilder(ENDPOINT) .GET() diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt index 471b6bc3..030e3f37 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt @@ -17,7 +17,7 @@ import javax.inject.Singleton public class RuneLiteKeyDownloader @Inject constructor( private val client: HttpClient, jsonKeyReader: JsonKeyReader -) : JsonKeyDownloader(client, jsonKeyReader) { +) : JsonKeyDownloader(KeySource.RUNELITE, client, jsonKeyReader) { override suspend fun getMissingUrls(seenUrls: Set): Set { val version = getVersion() return setOf(getXteaEndpoint(version)) diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/KeysController.kt b/archive/src/main/kotlin/org/openrs2/archive/web/KeysController.kt index 88ab3bbf..07e8f94c 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/KeysController.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/KeysController.kt @@ -1,14 +1,20 @@ package org.openrs2.archive.web import io.ktor.application.ApplicationCall +import io.ktor.http.HttpStatusCode +import io.ktor.request.receive import io.ktor.response.respond import io.ktor.thymeleaf.ThymeleafContent import org.openrs2.archive.key.KeyExporter +import org.openrs2.archive.key.KeyImporter +import org.openrs2.archive.key.KeySource +import org.openrs2.crypto.XteaKey import javax.inject.Inject import javax.inject.Singleton @Singleton public class KeysController @Inject constructor( + private val importer: KeyImporter, private val exporter: KeyExporter ) { public suspend fun index(call: ApplicationCall) { @@ -17,6 +23,16 @@ public class KeysController @Inject constructor( call.respond(ThymeleafContent("keys/index.html", mapOf("stats" to stats, "analysis" to analysis))) } + public suspend fun import(call: ApplicationCall) { + val keys = call.receive>().mapTo(mutableSetOf(), XteaKey::fromIntArray) + + if (keys.isNotEmpty()) { + importer.import(keys, KeySource.API) + } + + call.respond(HttpStatusCode.NoContent) + } + public suspend fun exportAll(call: ApplicationCall) { call.respond(exporter.exportAll()) } diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt index 80a70f10..8c5665c1 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt @@ -13,6 +13,7 @@ import io.ktor.jackson.JacksonConverter import io.ktor.response.respond import io.ktor.response.respondRedirect import io.ktor.routing.get +import io.ktor.routing.post import io.ktor.routing.routing import io.ktor.server.cio.CIO import io.ktor.server.engine.embeddedServer @@ -83,6 +84,7 @@ public class WebServer @Inject constructor( get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) } get("/caches/{id}/map.png") { cachesController.renderMap(call) } get("/keys") { keysController.index(call) } + post("/keys") { keysController.import(call) } get("/keys/all.json") { keysController.exportAll(call) } get("/keys/valid.json") { keysController.exportValid(call) } static("/static") { resources("/org/openrs2/archive/static") } diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V11__keys.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V11__keys.sql new file mode 100644 index 00000000..a75b22bf --- /dev/null +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V11__keys.sql @@ -0,0 +1,24 @@ +-- @formatter:off +CREATE TYPE key_source AS ENUM ( + 'api', + 'disk', + 'openosrs', + 'polar', + 'runelite' +); + +CREATE TABLE key_sources ( + key_id BIGINT NOT NULL REFERENCES keys (id), + source key_source NOT NULL, + first_seen TIMESTAMPTZ NOT NULL, + last_seen TIMESTAMPTZ NOT NULL, + PRIMARY KEY (key_id, source) +); + +CREATE TABLE key_queue ( + key xtea_key NOT NULL, + source key_source NOT NULL, + first_seen TIMESTAMPTZ NOT NULL, + last_seen TIMESTAMPTZ NOT NULL, + PRIMARY KEY (key, source) +);