Add support for downloading keys from Polar's archive

Signed-off-by: Graham <gpe@openrs2.org>
bzip2
Graham 4 years ago
parent b2ccbad031
commit 3a067b8b9c
  1. 1
      archive/build.gradle.kts
  2. 2
      archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt
  3. 31
      archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt
  4. 3
      archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt
  5. 128
      archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt
  6. 31
      archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt
  7. 48
      archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt
  8. 25
      archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt
  9. 4
      archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql
  10. 1
      buildSrc/src/main/kotlin/Versions.kt

@ -30,6 +30,7 @@ dependencies {
implementation("org.flywaydb:flyway-core:${Versions.flyway}")
implementation("org.jdom:jdom2:${Versions.jdom}")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
implementation("org.jsoup:jsoup:${Versions.jsoup}")
implementation("org.postgresql:postgresql:${Versions.postgres}")
implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}")
implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}")

@ -5,6 +5,7 @@ import com.google.inject.Scopes
import com.google.inject.multibindings.Multibinder
import org.openrs2.archive.key.KeyDownloader
import org.openrs2.archive.key.OpenOsrsKeyDownloader
import org.openrs2.archive.key.PolarKeyDownloader
import org.openrs2.archive.key.RuneLiteKeyDownloader
import org.openrs2.archive.name.NameDownloader
import org.openrs2.archive.name.RuneStarNameDownloader
@ -34,6 +35,7 @@ public object ArchiveModule : AbstractModule() {
val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java)
keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java)
keyBinder.addBinding().to(PolarKeyDownloader::class.java)
keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java)
val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java)

@ -0,0 +1,31 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
public abstract class JsonKeyDownloader(
private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader
) : KeyDownloader {
override suspend fun download(url: String): Sequence<XteaKey> {
val request = HttpRequest.newBuilder(URI(url))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
}
}

@ -3,5 +3,6 @@ package org.openrs2.archive.key
import org.openrs2.crypto.XteaKey
public interface KeyDownloader {
public suspend fun download(): Sequence<XteaKey>
public suspend fun getMissingUrls(seenUrls: Set<String>): Set<String>
public suspend fun download(url: String): Sequence<XteaKey>
}

@ -4,6 +4,7 @@ import org.openrs2.crypto.XteaKey
import org.openrs2.db.Database
import java.nio.file.Files
import java.nio.file.Path
import java.sql.Connection
import javax.inject.Inject
import javax.inject.Singleton
@ -40,68 +41,109 @@ public class KeyImporter @Inject constructor(
}
public suspend fun download() {
val keys = mutableSetOf<XteaKey>()
for (downloader in downloaders) {
keys += downloader.download()
}
keys -= XteaKey.ZERO
import(keys)
}
public suspend fun import(keys: Iterable<XteaKey>) {
database.execute { connection ->
val seenUrls = database.execute { connection ->
connection.prepareStatement(
"""
LOCK TABLE keys IN EXCLUSIVE MODE
SELECT url FROM keysets
""".trimIndent()
).use { stmt ->
stmt.execute()
stmt.executeQuery().use { rows ->
val urls = mutableSetOf<String>()
while (rows.next()) {
urls += rows.getString(1)
}
return@execute urls
}
}
}
connection.prepareStatement(
"""
CREATE TEMPORARY TABLE tmp_keys (
key xtea_key NOT NULL
) ON COMMIT DROP
""".trimIndent()
).use { stmt ->
stmt.execute()
val keys = mutableSetOf<XteaKey>()
val urls = mutableSetOf<String>()
for (downloader in downloaders) {
for (url in downloader.getMissingUrls(seenUrls)) {
keys += downloader.download(url)
urls += url
}
}
database.execute { connection ->
connection.prepareStatement(
"""
INSERT INTO tmp_keys (key)
VALUES (ROW(?, ?, ?, ?))
INSERT INTO keysets (url)
VALUES (?)
ON CONFLICT DO NOTHING
""".trimIndent()
).use { stmt ->
for (key in keys) {
if (key.isZero) {
continue
}
stmt.setInt(1, key.k0)
stmt.setInt(2, key.k1)
stmt.setInt(3, key.k2)
stmt.setInt(4, key.k3)
for (url in urls) {
stmt.setString(1, url)
stmt.addBatch()
}
stmt.executeBatch()
}
connection.prepareStatement(
"""
INSERT INTO keys (key)
SELECT t.key
FROM tmp_keys t
LEFT JOIN keys k ON k.key = t.key
WHERE k.key IS NULL
ON CONFLICT DO NOTHING
""".trimIndent()
).use { stmt ->
stmt.execute()
import(connection, keys)
}
}
public suspend fun import(keys: Iterable<XteaKey>) {
database.execute { connection ->
import(connection, keys)
}
}
private fun import(connection: Connection, keys: Iterable<XteaKey>) {
connection.prepareStatement(
"""
LOCK TABLE keys IN EXCLUSIVE MODE
""".trimIndent()
).use { stmt ->
stmt.execute()
}
connection.prepareStatement(
"""
CREATE TEMPORARY TABLE tmp_keys (
key xtea_key NOT NULL
) ON COMMIT DROP
""".trimIndent()
).use { stmt ->
stmt.execute()
}
connection.prepareStatement(
"""
INSERT INTO tmp_keys (key)
VALUES (ROW(?, ?, ?, ?))
""".trimIndent()
).use { stmt ->
for (key in keys) {
if (key.isZero) {
continue
}
stmt.setInt(1, key.k0)
stmt.setInt(2, key.k1)
stmt.setInt(3, key.k2)
stmt.setInt(4, key.k3)
stmt.addBatch()
}
stmt.executeBatch()
}
connection.prepareStatement(
"""
INSERT INTO keys (key)
SELECT t.key
FROM tmp_keys t
LEFT JOIN keys k ON k.key = t.key
WHERE k.key IS NULL
ON CONFLICT DO NOTHING
""".trimIndent()
).use { stmt ->
stmt.execute()
}
}
}

@ -1,38 +1,19 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
public class OpenOsrsKeyDownloader @Inject constructor(
private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader
) : KeyDownloader {
override suspend fun download(): Sequence<XteaKey> {
val request = HttpRequest.newBuilder(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
client: HttpClient,
jsonKeyReader: JsonKeyReader
) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
return setOf(ENDPOINT)
}
private companion object {
private val ENDPOINT = URI("https://xtea.openosrs.dev/get")
private const val ENDPOINT = "https://xtea.openosrs.dev/get"
}
}

@ -0,0 +1,48 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.jsoup.Jsoup
import org.openrs2.http.charset
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
public class PolarKeyDownloader @Inject constructor(
private val client: HttpClient,
jsonKeyReader: JsonKeyReader
) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val request = HttpRequest.newBuilder(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
val document = withContext(Dispatchers.IO) {
Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString())
}
val urls = mutableSetOf<String>()
for (element in document.select("a")) {
val url = element.absUrl("href")
if (url.endsWith(".json") && url !in seenUrls) {
urls += url
}
}
return urls
}
private companion object {
private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/")
}
}

@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.jdom2.input.SAXBuilder
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
@ -16,23 +15,11 @@ import javax.inject.Singleton
@Singleton
public class RuneLiteKeyDownloader @Inject constructor(
private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader
) : KeyDownloader {
override suspend fun download(): Sequence<XteaKey> {
jsonKeyReader: JsonKeyReader
) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val version = getVersion()
val request = HttpRequest.newBuilder(getXteaEndpoint(version))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
return setOf(getXteaEndpoint(version))
}
private suspend fun getVersion(): String {
@ -58,8 +45,8 @@ public class RuneLiteKeyDownloader @Inject constructor(
private companion object {
private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml")
private fun getXteaEndpoint(version: String): URI {
return URI("https://api.runelite.net/runelite-$version/xtea")
private fun getXteaEndpoint(version: String): String {
return "https://api.runelite.net/runelite-$version/xtea"
}
}
}

@ -26,6 +26,10 @@ CREATE TABLE keys (
key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0)
);
CREATE TABLE keysets (
url TEXT PRIMARY KEY NOT NULL
);
CREATE TABLE containers (
id BIGSERIAL PRIMARY KEY NOT NULL,
crc32 INTEGER NOT NULL,

@ -20,6 +20,7 @@ object Versions {
const val jdom = "2.0.6"
const val jgrapht = "1.5.0"
const val jimfs = "1.2"
const val jsoup = "1.13.1"
const val junit = "5.7.1"
const val kotlin = "1.4.30"
const val kotlinCoroutines = "1.4.2"

Loading…
Cancel
Save