Add support for downloading keys from Polar's archive

Signed-off-by: Graham <gpe@openrs2.org>

archive/build.gradle.kts

@@ -30,6 +30,7 @@ dependencies {
     implementation("org.flywaydb:flyway-core:${Versions.flyway}")
     implementation("org.jdom:jdom2:${Versions.jdom}")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
+    implementation("org.jsoup:jsoup:${Versions.jsoup}")
     implementation("org.postgresql:postgresql:${Versions.postgres}")
     implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}")
     implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}")

archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt

@@ -5,6 +5,7 @@ import com.google.inject.Scopes
 import com.google.inject.multibindings.Multibinder
 import org.openrs2.archive.key.KeyDownloader
 import org.openrs2.archive.key.OpenOsrsKeyDownloader
+import org.openrs2.archive.key.PolarKeyDownloader
 import org.openrs2.archive.key.RuneLiteKeyDownloader
 import org.openrs2.archive.name.NameDownloader
 import org.openrs2.archive.name.RuneStarNameDownloader
@@ -34,6 +35,7 @@ public object ArchiveModule : AbstractModule() {
 
         val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java)
         keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java)
+        keyBinder.addBinding().to(PolarKeyDownloader::class.java)
         keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java)
 
         val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java)

archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt

@@ -0,0 +1,31 @@
+package org.openrs2.archive.key
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.future.await
+import kotlinx.coroutines.withContext
+import org.openrs2.crypto.XteaKey
+import org.openrs2.http.checkStatusCode
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+
+public abstract class JsonKeyDownloader(
+    private val client: HttpClient,
+    private val jsonKeyReader: JsonKeyReader
+) : KeyDownloader {
+    override suspend fun download(url: String): Sequence<XteaKey> {
+        val request = HttpRequest.newBuilder(URI(url))
+            .GET()
+            .build()
+
+        val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
+        response.checkStatusCode()
+
+        return withContext(Dispatchers.IO) {
+            response.body().use { input ->
+                jsonKeyReader.read(input)
+            }
+        }
+    }
+}

archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt

@@ -3,5 +3,6 @@ package org.openrs2.archive.key
 import org.openrs2.crypto.XteaKey
 
 public interface KeyDownloader {
-    public suspend fun download(): Sequence<XteaKey>
+    public suspend fun getMissingUrls(seenUrls: Set<String>): Set<String>
+    public suspend fun download(url: String): Sequence<XteaKey>
 }

archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt

@@ -4,6 +4,7 @@ import org.openrs2.crypto.XteaKey
 import org.openrs2.db.Database
 import java.nio.file.Files
 import java.nio.file.Path
+import java.sql.Connection
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -40,68 +41,109 @@ public class KeyImporter @Inject constructor(
     }
 
     public suspend fun download() {
-        val keys = mutableSetOf<XteaKey>()
-        for (downloader in downloaders) {
-            keys += downloader.download()
-        }
-        keys -= XteaKey.ZERO
-
-        import(keys)
-    }
-
-    public suspend fun import(keys: Iterable<XteaKey>) {
-        database.execute { connection ->
+        val seenUrls = database.execute { connection ->
             connection.prepareStatement(
                 """
-                LOCK TABLE keys IN EXCLUSIVE MODE
+                SELECT url FROM keysets
             """.trimIndent()
             ).use { stmt ->
-                stmt.execute()
+                stmt.executeQuery().use { rows ->
+                    val urls = mutableSetOf<String>()
+                    while (rows.next()) {
+                        urls += rows.getString(1)
+                    }
+                    return@execute urls
+                }
             }
+        }
 
-            connection.prepareStatement(
-                """
-                CREATE TEMPORARY TABLE tmp_keys (
-                    key xtea_key NOT NULL
-                ) ON COMMIT DROP
-            """.trimIndent()
-            ).use { stmt ->
-                stmt.execute()
+        val keys = mutableSetOf<XteaKey>()
+        val urls = mutableSetOf<String>()
+
+        for (downloader in downloaders) {
+            for (url in downloader.getMissingUrls(seenUrls)) {
+                keys += downloader.download(url)
+                urls += url
             }
+        }
 
+        database.execute { connection ->
             connection.prepareStatement(
                 """
-                INSERT INTO tmp_keys (key)
-                VALUES (ROW(?, ?, ?, ?))
+                INSERT INTO keysets (url)
+                VALUES (?)
+                ON CONFLICT DO NOTHING
             """.trimIndent()
             ).use { stmt ->
-                for (key in keys) {
-                    if (key.isZero) {
-                        continue
-                    }
-
-                    stmt.setInt(1, key.k0)
-                    stmt.setInt(2, key.k1)
-                    stmt.setInt(3, key.k2)
-                    stmt.setInt(4, key.k3)
+                for (url in urls) {
+                    stmt.setString(1, url)
                     stmt.addBatch()
                 }
 
                 stmt.executeBatch()
             }
 
-            connection.prepareStatement(
-                """
-                INSERT INTO keys (key)
-                SELECT t.key
-                FROM tmp_keys t
-                LEFT JOIN keys k ON k.key = t.key
-                WHERE k.key IS NULL
-                ON CONFLICT DO NOTHING
-            """.trimIndent()
-            ).use { stmt ->
-                stmt.execute()
+            import(connection, keys)
+        }
+    }
+
+    public suspend fun import(keys: Iterable<XteaKey>) {
+        database.execute { connection ->
+            import(connection, keys)
+        }
+    }
+
+    private fun import(connection: Connection, keys: Iterable<XteaKey>) {
+        connection.prepareStatement(
+            """
+            LOCK TABLE keys IN EXCLUSIVE MODE
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.execute()
+        }
+
+        connection.prepareStatement(
+            """
+            CREATE TEMPORARY TABLE tmp_keys (
+                key xtea_key NOT NULL
+            ) ON COMMIT DROP
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.execute()
+        }
+
+        connection.prepareStatement(
+            """
+            INSERT INTO tmp_keys (key)
+            VALUES (ROW(?, ?, ?, ?))
+        """.trimIndent()
+        ).use { stmt ->
+            for (key in keys) {
+                if (key.isZero) {
+                    continue
+                }
+
+                stmt.setInt(1, key.k0)
+                stmt.setInt(2, key.k1)
+                stmt.setInt(3, key.k2)
+                stmt.setInt(4, key.k3)
+                stmt.addBatch()
             }
+
+            stmt.executeBatch()
+        }
+
+        connection.prepareStatement(
+            """
+            INSERT INTO keys (key)
+            SELECT t.key
+            FROM tmp_keys t
+            LEFT JOIN keys k ON k.key = t.key
+            WHERE k.key IS NULL
+            ON CONFLICT DO NOTHING
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.execute()
         }
     }
 }

archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt

@@ -1,38 +1,19 @@
 package org.openrs2.archive.key
 
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.future.await
-import kotlinx.coroutines.withContext
-import org.openrs2.crypto.XteaKey
-import org.openrs2.http.checkStatusCode
-import java.net.URI
 import java.net.http.HttpClient
-import java.net.http.HttpRequest
-import java.net.http.HttpResponse
 import javax.inject.Inject
 import javax.inject.Singleton
 
 @Singleton
 public class OpenOsrsKeyDownloader @Inject constructor(
-    private val client: HttpClient,
-    private val jsonKeyReader: JsonKeyReader
-) : KeyDownloader {
-    override suspend fun download(): Sequence<XteaKey> {
-        val request = HttpRequest.newBuilder(ENDPOINT)
-            .GET()
-            .build()
-
-        val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
-        response.checkStatusCode()
-
-        return withContext(Dispatchers.IO) {
-            response.body().use { input ->
-                jsonKeyReader.read(input)
-            }
-        }
+    client: HttpClient,
+    jsonKeyReader: JsonKeyReader
+) : JsonKeyDownloader(client, jsonKeyReader) {
+    override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
+        return setOf(ENDPOINT)
     }
 
     private companion object {
-        private val ENDPOINT = URI("https://xtea.openosrs.dev/get")
+        private const val ENDPOINT = "https://xtea.openosrs.dev/get"
     }
 }

archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt

@@ -0,0 +1,48 @@
+package org.openrs2.archive.key
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.future.await
+import kotlinx.coroutines.withContext
+import org.jsoup.Jsoup
+import org.openrs2.http.charset
+import org.openrs2.http.checkStatusCode
+import java.net.URI
+import java.net.http.HttpClient
+import java.net.http.HttpRequest
+import java.net.http.HttpResponse
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+public class PolarKeyDownloader @Inject constructor(
+    private val client: HttpClient,
+    jsonKeyReader: JsonKeyReader
+) : JsonKeyDownloader(client, jsonKeyReader) {
+    override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
+        val request = HttpRequest.newBuilder(ENDPOINT)
+            .GET()
+            .build()
+
+        val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
+        response.checkStatusCode()
+
+        val document = withContext(Dispatchers.IO) {
+            Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString())
+        }
+
+        val urls = mutableSetOf<String>()
+
+        for (element in document.select("a")) {
+            val url = element.absUrl("href")
+            if (url.endsWith(".json") && url !in seenUrls) {
+                urls += url
+            }
+        }
+
+        return urls
+    }
+
+    private companion object {
+        private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/")
+    }
+}

archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt

@@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.future.await
 import kotlinx.coroutines.withContext
 import org.jdom2.input.SAXBuilder
-import org.openrs2.crypto.XteaKey
 import org.openrs2.http.checkStatusCode
 import java.net.URI
 import java.net.http.HttpClient
@@ -16,23 +15,11 @@ import javax.inject.Singleton
 @Singleton
 public class RuneLiteKeyDownloader @Inject constructor(
     private val client: HttpClient,
-    private val jsonKeyReader: JsonKeyReader
-) : KeyDownloader {
-    override suspend fun download(): Sequence<XteaKey> {
+    jsonKeyReader: JsonKeyReader
+) : JsonKeyDownloader(client, jsonKeyReader) {
+    override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
         val version = getVersion()
-
-        val request = HttpRequest.newBuilder(getXteaEndpoint(version))
-            .GET()
-            .build()
-
-        val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
-        response.checkStatusCode()
-
-        return withContext(Dispatchers.IO) {
-            response.body().use { input ->
-                jsonKeyReader.read(input)
-            }
-        }
+        return setOf(getXteaEndpoint(version))
     }
 
     private suspend fun getVersion(): String {
@@ -58,8 +45,8 @@ public class RuneLiteKeyDownloader @Inject constructor(
     private companion object {
         private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml")
 
-        private fun getXteaEndpoint(version: String): URI {
-            return URI("https://api.runelite.net/runelite-$version/xtea")
+        private fun getXteaEndpoint(version: String): String {
+            return "https://api.runelite.net/runelite-$version/xtea"
         }
     }
 }

archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql

@@ -26,6 +26,10 @@ CREATE TABLE keys (
     key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0)
 );
 
+CREATE TABLE keysets (
+    url TEXT PRIMARY KEY NOT NULL
+);
+
 CREATE TABLE containers (
     id BIGSERIAL PRIMARY KEY NOT NULL,
     crc32 INTEGER NOT NULL,

buildSrc/src/main/kotlin/Versions.kt

@@ -20,6 +20,7 @@ object Versions {
     const val jdom = "2.0.6"
     const val jgrapht = "1.5.0"
     const val jimfs = "1.2"
+    const val jsoup = "1.13.1"
     const val junit = "5.7.1"
     const val kotlin = "1.4.30"
     const val kotlinCoroutines = "1.4.2"