diff --git a/archive/build.gradle.kts b/archive/build.gradle.kts index 56abf5ebea..9f8f255cde 100644 --- a/archive/build.gradle.kts +++ b/archive/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { implementation("org.flywaydb:flyway-core:${Versions.flyway}") implementation("org.jdom:jdom2:${Versions.jdom}") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}") + implementation("org.jsoup:jsoup:${Versions.jsoup}") implementation("org.postgresql:postgresql:${Versions.postgres}") implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}") implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}") diff --git a/archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt b/archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt index c690b61ec4..484f339ad5 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt @@ -5,6 +5,7 @@ import com.google.inject.Scopes import com.google.inject.multibindings.Multibinder import org.openrs2.archive.key.KeyDownloader import org.openrs2.archive.key.OpenOsrsKeyDownloader +import org.openrs2.archive.key.PolarKeyDownloader import org.openrs2.archive.key.RuneLiteKeyDownloader import org.openrs2.archive.name.NameDownloader import org.openrs2.archive.name.RuneStarNameDownloader @@ -34,6 +35,7 @@ public object ArchiveModule : AbstractModule() { val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java) keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java) + keyBinder.addBinding().to(PolarKeyDownloader::class.java) keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java) val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java) diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt new file mode 100644 index 0000000000..4a8f9a8604 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt @@ -0,0 +1,31 @@ +package org.openrs2.archive.key + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.future.await +import kotlinx.coroutines.withContext +import org.openrs2.crypto.XteaKey +import org.openrs2.http.checkStatusCode +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse + +public abstract class JsonKeyDownloader( + private val client: HttpClient, + private val jsonKeyReader: JsonKeyReader +) : KeyDownloader { + override suspend fun download(url: String): Sequence { + val request = HttpRequest.newBuilder(URI(url)) + .GET() + .build() + + val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() + response.checkStatusCode() + + return withContext(Dispatchers.IO) { + response.body().use { input -> + jsonKeyReader.read(input) + } + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt index d2e1a916ef..2de4a4789a 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt @@ -3,5 +3,6 @@ package org.openrs2.archive.key import org.openrs2.crypto.XteaKey public interface KeyDownloader { - public suspend fun download(): Sequence + public suspend fun getMissingUrls(seenUrls: Set): Set + public suspend fun download(url: String): Sequence } diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt index 9b9e3cfe93..1813f777e4 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt @@ -4,6 +4,7 @@ import org.openrs2.crypto.XteaKey import org.openrs2.db.Database import java.nio.file.Files import java.nio.file.Path +import java.sql.Connection import javax.inject.Inject import javax.inject.Singleton @@ -40,68 +41,109 @@ public class KeyImporter @Inject constructor( } public suspend fun download() { - val keys = mutableSetOf() - for (downloader in downloaders) { - keys += downloader.download() - } - keys -= XteaKey.ZERO - - import(keys) - } - - public suspend fun import(keys: Iterable) { - database.execute { connection -> + val seenUrls = database.execute { connection -> connection.prepareStatement( """ - LOCK TABLE keys IN EXCLUSIVE MODE + SELECT url FROM keysets """.trimIndent() ).use { stmt -> - stmt.execute() + stmt.executeQuery().use { rows -> + val urls = mutableSetOf() + while (rows.next()) { + urls += rows.getString(1) + } + return@execute urls + } } + } - connection.prepareStatement( - """ - CREATE TEMPORARY TABLE tmp_keys ( - key xtea_key NOT NULL - ) ON COMMIT DROP - """.trimIndent() - ).use { stmt -> - stmt.execute() + val keys = mutableSetOf() + val urls = mutableSetOf() + + for (downloader in downloaders) { + for (url in downloader.getMissingUrls(seenUrls)) { + keys += downloader.download(url) + urls += url } + } + database.execute { connection -> connection.prepareStatement( """ - INSERT INTO tmp_keys (key) - VALUES (ROW(?, ?, ?, ?)) + INSERT INTO keysets (url) + VALUES (?) + ON CONFLICT DO NOTHING """.trimIndent() ).use { stmt -> - for (key in keys) { - if (key.isZero) { - continue - } - - stmt.setInt(1, key.k0) - stmt.setInt(2, key.k1) - stmt.setInt(3, key.k2) - stmt.setInt(4, key.k3) + for (url in urls) { + stmt.setString(1, url) stmt.addBatch() } stmt.executeBatch() } - connection.prepareStatement( - """ - INSERT INTO keys (key) - SELECT t.key - FROM tmp_keys t - LEFT JOIN keys k ON k.key = t.key - WHERE k.key IS NULL - ON CONFLICT DO NOTHING - """.trimIndent() - ).use { stmt -> - stmt.execute() + import(connection, keys) + } + } + + public suspend fun import(keys: Iterable) { + database.execute { connection -> + import(connection, keys) + } + } + + private fun import(connection: Connection, keys: Iterable) { + connection.prepareStatement( + """ + LOCK TABLE keys IN EXCLUSIVE MODE + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + CREATE TEMPORARY TABLE tmp_keys ( + key xtea_key NOT NULL + ) ON COMMIT DROP + """.trimIndent() + ).use { stmt -> + stmt.execute() + } + + connection.prepareStatement( + """ + INSERT INTO tmp_keys (key) + VALUES (ROW(?, ?, ?, ?)) + """.trimIndent() + ).use { stmt -> + for (key in keys) { + if (key.isZero) { + continue + } + + stmt.setInt(1, key.k0) + stmt.setInt(2, key.k1) + stmt.setInt(3, key.k2) + stmt.setInt(4, key.k3) + stmt.addBatch() } + + stmt.executeBatch() + } + + connection.prepareStatement( + """ + INSERT INTO keys (key) + SELECT t.key + FROM tmp_keys t + LEFT JOIN keys k ON k.key = t.key + WHERE k.key IS NULL + ON CONFLICT DO NOTHING + """.trimIndent() + ).use { stmt -> + stmt.execute() } } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt index aec6c896fa..8bb88805aa 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt @@ -1,38 +1,19 @@ package org.openrs2.archive.key -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.await -import kotlinx.coroutines.withContext -import org.openrs2.crypto.XteaKey -import org.openrs2.http.checkStatusCode -import java.net.URI import java.net.http.HttpClient -import java.net.http.HttpRequest -import java.net.http.HttpResponse import javax.inject.Inject import javax.inject.Singleton @Singleton public class OpenOsrsKeyDownloader @Inject constructor( - private val client: HttpClient, - private val jsonKeyReader: JsonKeyReader -) : KeyDownloader { - override suspend fun download(): Sequence { - val request = HttpRequest.newBuilder(ENDPOINT) - .GET() - .build() - - val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() - response.checkStatusCode() - - return withContext(Dispatchers.IO) { - response.body().use { input -> - jsonKeyReader.read(input) - } - } + client: HttpClient, + jsonKeyReader: JsonKeyReader +) : JsonKeyDownloader(client, jsonKeyReader) { + override suspend fun getMissingUrls(seenUrls: Set): Set { + return setOf(ENDPOINT) } private companion object { - private val ENDPOINT = URI("https://xtea.openosrs.dev/get") + private const val ENDPOINT = "https://xtea.openosrs.dev/get" } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt new file mode 100644 index 0000000000..dca63683df --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt @@ -0,0 +1,48 @@ +package org.openrs2.archive.key + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.future.await +import kotlinx.coroutines.withContext +import org.jsoup.Jsoup +import org.openrs2.http.charset +import org.openrs2.http.checkStatusCode +import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import javax.inject.Inject +import javax.inject.Singleton + +@Singleton +public class PolarKeyDownloader @Inject constructor( + private val client: HttpClient, + jsonKeyReader: JsonKeyReader +) : JsonKeyDownloader(client, jsonKeyReader) { + override suspend fun getMissingUrls(seenUrls: Set): Set { + val request = HttpRequest.newBuilder(ENDPOINT) + .GET() + .build() + + val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() + response.checkStatusCode() + + val document = withContext(Dispatchers.IO) { + Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString()) + } + + val urls = mutableSetOf() + + for (element in document.select("a")) { + val url = element.absUrl("href") + if (url.endsWith(".json") && url !in seenUrls) { + urls += url + } + } + + return urls + } + + private companion object { + private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/") + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt b/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt index 0f037b9377..a86008633d 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt @@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.future.await import kotlinx.coroutines.withContext import org.jdom2.input.SAXBuilder -import org.openrs2.crypto.XteaKey import org.openrs2.http.checkStatusCode import java.net.URI import java.net.http.HttpClient @@ -16,23 +15,11 @@ import javax.inject.Singleton @Singleton public class RuneLiteKeyDownloader @Inject constructor( private val client: HttpClient, - private val jsonKeyReader: JsonKeyReader -) : KeyDownloader { - override suspend fun download(): Sequence { + jsonKeyReader: JsonKeyReader +) : JsonKeyDownloader(client, jsonKeyReader) { + override suspend fun getMissingUrls(seenUrls: Set): Set { val version = getVersion() - - val request = HttpRequest.newBuilder(getXteaEndpoint(version)) - .GET() - .build() - - val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() - response.checkStatusCode() - - return withContext(Dispatchers.IO) { - response.body().use { input -> - jsonKeyReader.read(input) - } - } + return setOf(getXteaEndpoint(version)) } private suspend fun getVersion(): String { @@ -58,8 +45,8 @@ public class RuneLiteKeyDownloader @Inject constructor( private companion object { private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml") - private fun getXteaEndpoint(version: String): URI { - return URI("https://api.runelite.net/runelite-$version/xtea") + private fun getXteaEndpoint(version: String): String { + return "https://api.runelite.net/runelite-$version/xtea" } } } diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql index 7596edab91..1cb00e1304 100644 --- a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql @@ -26,6 +26,10 @@ CREATE TABLE keys ( key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0) ); +CREATE TABLE keysets ( + url TEXT PRIMARY KEY NOT NULL +); + CREATE TABLE containers ( id BIGSERIAL PRIMARY KEY NOT NULL, crc32 INTEGER NOT NULL, diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index a8195d34e0..fc25fe43ed 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -20,6 +20,7 @@ object Versions { const val jdom = "2.0.6" const val jgrapht = "1.5.0" const val jimfs = "1.2" + const val jsoup = "1.13.1" const val junit = "5.7.1" const val kotlin = "1.4.30" const val kotlinCoroutines = "1.4.2"