Add support for downloading keys from Polar's archive

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent b2ccbad031
commit 3a067b8b9c
  1. 1
      archive/build.gradle.kts
  2. 2
      archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt
  3. 31
      archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt
  4. 3
      archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt
  5. 128
      archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt
  6. 31
      archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt
  7. 48
      archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt
  8. 25
      archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt
  9. 4
      archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql
  10. 1
      buildSrc/src/main/kotlin/Versions.kt

@ -30,6 +30,7 @@ dependencies {
implementation("org.flywaydb:flyway-core:${Versions.flyway}") implementation("org.flywaydb:flyway-core:${Versions.flyway}")
implementation("org.jdom:jdom2:${Versions.jdom}") implementation("org.jdom:jdom2:${Versions.jdom}")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
implementation("org.jsoup:jsoup:${Versions.jsoup}")
implementation("org.postgresql:postgresql:${Versions.postgres}") implementation("org.postgresql:postgresql:${Versions.postgres}")
implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}") implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}")
implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}") implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}")

@ -5,6 +5,7 @@ import com.google.inject.Scopes
import com.google.inject.multibindings.Multibinder import com.google.inject.multibindings.Multibinder
import org.openrs2.archive.key.KeyDownloader import org.openrs2.archive.key.KeyDownloader
import org.openrs2.archive.key.OpenOsrsKeyDownloader import org.openrs2.archive.key.OpenOsrsKeyDownloader
import org.openrs2.archive.key.PolarKeyDownloader
import org.openrs2.archive.key.RuneLiteKeyDownloader import org.openrs2.archive.key.RuneLiteKeyDownloader
import org.openrs2.archive.name.NameDownloader import org.openrs2.archive.name.NameDownloader
import org.openrs2.archive.name.RuneStarNameDownloader import org.openrs2.archive.name.RuneStarNameDownloader
@ -34,6 +35,7 @@ public object ArchiveModule : AbstractModule() {
val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java) val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java)
keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java) keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java)
keyBinder.addBinding().to(PolarKeyDownloader::class.java)
keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java) keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java)
val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java) val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java)

@ -0,0 +1,31 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
public abstract class JsonKeyDownloader(
private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader
) : KeyDownloader {
override suspend fun download(url: String): Sequence<XteaKey> {
val request = HttpRequest.newBuilder(URI(url))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
}
}

@ -3,5 +3,6 @@ package org.openrs2.archive.key
import org.openrs2.crypto.XteaKey import org.openrs2.crypto.XteaKey
public interface KeyDownloader { public interface KeyDownloader {
public suspend fun download(): Sequence<XteaKey> public suspend fun getMissingUrls(seenUrls: Set<String>): Set<String>
public suspend fun download(url: String): Sequence<XteaKey>
} }

@ -4,6 +4,7 @@ import org.openrs2.crypto.XteaKey
import org.openrs2.db.Database import org.openrs2.db.Database
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.sql.Connection
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -40,68 +41,109 @@ public class KeyImporter @Inject constructor(
} }
public suspend fun download() { public suspend fun download() {
val keys = mutableSetOf<XteaKey>() val seenUrls = database.execute { connection ->
for (downloader in downloaders) {
keys += downloader.download()
}
keys -= XteaKey.ZERO
import(keys)
}
public suspend fun import(keys: Iterable<XteaKey>) {
database.execute { connection ->
connection.prepareStatement( connection.prepareStatement(
""" """
LOCK TABLE keys IN EXCLUSIVE MODE SELECT url FROM keysets
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.execute() stmt.executeQuery().use { rows ->
val urls = mutableSetOf<String>()
while (rows.next()) {
urls += rows.getString(1)
}
return@execute urls
}
} }
}
connection.prepareStatement( val keys = mutableSetOf<XteaKey>()
""" val urls = mutableSetOf<String>()
CREATE TEMPORARY TABLE tmp_keys (
key xtea_key NOT NULL for (downloader in downloaders) {
) ON COMMIT DROP for (url in downloader.getMissingUrls(seenUrls)) {
""".trimIndent() keys += downloader.download(url)
).use { stmt -> urls += url
stmt.execute()
} }
}
database.execute { connection ->
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO tmp_keys (key) INSERT INTO keysets (url)
VALUES (ROW(?, ?, ?, ?)) VALUES (?)
ON CONFLICT DO NOTHING
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
for (key in keys) { for (url in urls) {
if (key.isZero) { stmt.setString(1, url)
continue
}
stmt.setInt(1, key.k0)
stmt.setInt(2, key.k1)
stmt.setInt(3, key.k2)
stmt.setInt(4, key.k3)
stmt.addBatch() stmt.addBatch()
} }
stmt.executeBatch() stmt.executeBatch()
} }
connection.prepareStatement( import(connection, keys)
""" }
INSERT INTO keys (key) }
SELECT t.key
FROM tmp_keys t public suspend fun import(keys: Iterable<XteaKey>) {
LEFT JOIN keys k ON k.key = t.key database.execute { connection ->
WHERE k.key IS NULL import(connection, keys)
ON CONFLICT DO NOTHING }
""".trimIndent() }
).use { stmt ->
stmt.execute() private fun import(connection: Connection, keys: Iterable<XteaKey>) {
connection.prepareStatement(
"""
LOCK TABLE keys IN EXCLUSIVE MODE
""".trimIndent()
).use { stmt ->
stmt.execute()
}
connection.prepareStatement(
"""
CREATE TEMPORARY TABLE tmp_keys (
key xtea_key NOT NULL
) ON COMMIT DROP
""".trimIndent()
).use { stmt ->
stmt.execute()
}
connection.prepareStatement(
"""
INSERT INTO tmp_keys (key)
VALUES (ROW(?, ?, ?, ?))
""".trimIndent()
).use { stmt ->
for (key in keys) {
if (key.isZero) {
continue
}
stmt.setInt(1, key.k0)
stmt.setInt(2, key.k1)
stmt.setInt(3, key.k2)
stmt.setInt(4, key.k3)
stmt.addBatch()
} }
stmt.executeBatch()
}
connection.prepareStatement(
"""
INSERT INTO keys (key)
SELECT t.key
FROM tmp_keys t
LEFT JOIN keys k ON k.key = t.key
WHERE k.key IS NULL
ON CONFLICT DO NOTHING
""".trimIndent()
).use { stmt ->
stmt.execute()
} }
} }
} }

@ -1,38 +1,19 @@
package org.openrs2.archive.key package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@Singleton @Singleton
public class OpenOsrsKeyDownloader @Inject constructor( public class OpenOsrsKeyDownloader @Inject constructor(
private val client: HttpClient, client: HttpClient,
private val jsonKeyReader: JsonKeyReader jsonKeyReader: JsonKeyReader
) : KeyDownloader { ) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun download(): Sequence<XteaKey> { override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val request = HttpRequest.newBuilder(ENDPOINT) return setOf(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
} }
private companion object { private companion object {
private val ENDPOINT = URI("https://xtea.openosrs.dev/get") private const val ENDPOINT = "https://xtea.openosrs.dev/get"
} }
} }

@ -0,0 +1,48 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.jsoup.Jsoup
import org.openrs2.http.charset
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
public class PolarKeyDownloader @Inject constructor(
private val client: HttpClient,
jsonKeyReader: JsonKeyReader
) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val request = HttpRequest.newBuilder(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
val document = withContext(Dispatchers.IO) {
Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString())
}
val urls = mutableSetOf<String>()
for (element in document.select("a")) {
val url = element.absUrl("href")
if (url.endsWith(".json") && url !in seenUrls) {
urls += url
}
}
return urls
}
private companion object {
private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/")
}
}

@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.jdom2.input.SAXBuilder import org.jdom2.input.SAXBuilder
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode import org.openrs2.http.checkStatusCode
import java.net.URI import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
@ -16,23 +15,11 @@ import javax.inject.Singleton
@Singleton @Singleton
public class RuneLiteKeyDownloader @Inject constructor( public class RuneLiteKeyDownloader @Inject constructor(
private val client: HttpClient, private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader jsonKeyReader: JsonKeyReader
) : KeyDownloader { ) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun download(): Sequence<XteaKey> { override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val version = getVersion() val version = getVersion()
return setOf(getXteaEndpoint(version))
val request = HttpRequest.newBuilder(getXteaEndpoint(version))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
} }
private suspend fun getVersion(): String { private suspend fun getVersion(): String {
@ -58,8 +45,8 @@ public class RuneLiteKeyDownloader @Inject constructor(
private companion object { private companion object {
private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml") private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml")
private fun getXteaEndpoint(version: String): URI { private fun getXteaEndpoint(version: String): String {
return URI("https://api.runelite.net/runelite-$version/xtea") return "https://api.runelite.net/runelite-$version/xtea"
} }
} }
} }

@ -26,6 +26,10 @@ CREATE TABLE keys (
key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0) key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0)
); );
CREATE TABLE keysets (
url TEXT PRIMARY KEY NOT NULL
);
CREATE TABLE containers ( CREATE TABLE containers (
id BIGSERIAL PRIMARY KEY NOT NULL, id BIGSERIAL PRIMARY KEY NOT NULL,
crc32 INTEGER NOT NULL, crc32 INTEGER NOT NULL,

@ -20,6 +20,7 @@ object Versions {
const val jdom = "2.0.6" const val jdom = "2.0.6"
const val jgrapht = "1.5.0" const val jgrapht = "1.5.0"
const val jimfs = "1.2" const val jimfs = "1.2"
const val jsoup = "1.13.1"
const val junit = "5.7.1" const val junit = "5.7.1"
const val kotlin = "1.4.30" const val kotlin = "1.4.30"
const val kotlinCoroutines = "1.4.2" const val kotlinCoroutines = "1.4.2"

Loading…
Cancel
Save