Add support for downloading keys from Polar's archive

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent b2ccbad031
commit 3a067b8b9c
  1. 1
      archive/build.gradle.kts
  2. 2
      archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt
  3. 31
      archive/src/main/kotlin/org/openrs2/archive/key/JsonKeyDownloader.kt
  4. 3
      archive/src/main/kotlin/org/openrs2/archive/key/KeyDownloader.kt
  5. 50
      archive/src/main/kotlin/org/openrs2/archive/key/KeyImporter.kt
  6. 31
      archive/src/main/kotlin/org/openrs2/archive/key/OpenOsrsKeyDownloader.kt
  7. 48
      archive/src/main/kotlin/org/openrs2/archive/key/PolarKeyDownloader.kt
  8. 25
      archive/src/main/kotlin/org/openrs2/archive/key/RuneLiteKeyDownloader.kt
  9. 4
      archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql
  10. 1
      buildSrc/src/main/kotlin/Versions.kt

@ -30,6 +30,7 @@ dependencies {
implementation("org.flywaydb:flyway-core:${Versions.flyway}") implementation("org.flywaydb:flyway-core:${Versions.flyway}")
implementation("org.jdom:jdom2:${Versions.jdom}") implementation("org.jdom:jdom2:${Versions.jdom}")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
implementation("org.jsoup:jsoup:${Versions.jsoup}")
implementation("org.postgresql:postgresql:${Versions.postgres}") implementation("org.postgresql:postgresql:${Versions.postgres}")
implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}") implementation("org.thymeleaf:thymeleaf:${Versions.thymeleaf}")
implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}") implementation("org.thymeleaf.extras:thymeleaf-extras-java8time:${Versions.thymeleafJava8Time}")

@ -5,6 +5,7 @@ import com.google.inject.Scopes
import com.google.inject.multibindings.Multibinder import com.google.inject.multibindings.Multibinder
import org.openrs2.archive.key.KeyDownloader import org.openrs2.archive.key.KeyDownloader
import org.openrs2.archive.key.OpenOsrsKeyDownloader import org.openrs2.archive.key.OpenOsrsKeyDownloader
import org.openrs2.archive.key.PolarKeyDownloader
import org.openrs2.archive.key.RuneLiteKeyDownloader import org.openrs2.archive.key.RuneLiteKeyDownloader
import org.openrs2.archive.name.NameDownloader import org.openrs2.archive.name.NameDownloader
import org.openrs2.archive.name.RuneStarNameDownloader import org.openrs2.archive.name.RuneStarNameDownloader
@ -34,6 +35,7 @@ public object ArchiveModule : AbstractModule() {
val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java) val keyBinder = Multibinder.newSetBinder(binder(), KeyDownloader::class.java)
keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java) keyBinder.addBinding().to(OpenOsrsKeyDownloader::class.java)
keyBinder.addBinding().to(PolarKeyDownloader::class.java)
keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java) keyBinder.addBinding().to(RuneLiteKeyDownloader::class.java)
val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java) val nameBinder = Multibinder.newSetBinder(binder(), NameDownloader::class.java)

@ -0,0 +1,31 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
public abstract class JsonKeyDownloader(
private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader
) : KeyDownloader {
override suspend fun download(url: String): Sequence<XteaKey> {
val request = HttpRequest.newBuilder(URI(url))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
}
}

@ -3,5 +3,6 @@ package org.openrs2.archive.key
import org.openrs2.crypto.XteaKey import org.openrs2.crypto.XteaKey
public interface KeyDownloader { public interface KeyDownloader {
public suspend fun download(): Sequence<XteaKey> public suspend fun getMissingUrls(seenUrls: Set<String>): Set<String>
public suspend fun download(url: String): Sequence<XteaKey>
} }

@ -4,6 +4,7 @@ import org.openrs2.crypto.XteaKey
import org.openrs2.db.Database import org.openrs2.db.Database
import java.nio.file.Files import java.nio.file.Files
import java.nio.file.Path import java.nio.file.Path
import java.sql.Connection
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@ -40,17 +41,59 @@ public class KeyImporter @Inject constructor(
} }
public suspend fun download() { public suspend fun download() {
val seenUrls = database.execute { connection ->
connection.prepareStatement(
"""
SELECT url FROM keysets
""".trimIndent()
).use { stmt ->
stmt.executeQuery().use { rows ->
val urls = mutableSetOf<String>()
while (rows.next()) {
urls += rows.getString(1)
}
return@execute urls
}
}
}
val keys = mutableSetOf<XteaKey>() val keys = mutableSetOf<XteaKey>()
val urls = mutableSetOf<String>()
for (downloader in downloaders) { for (downloader in downloaders) {
keys += downloader.download() for (url in downloader.getMissingUrls(seenUrls)) {
keys += downloader.download(url)
urls += url
}
} }
keys -= XteaKey.ZERO
import(keys) database.execute { connection ->
connection.prepareStatement(
"""
INSERT INTO keysets (url)
VALUES (?)
ON CONFLICT DO NOTHING
""".trimIndent()
).use { stmt ->
for (url in urls) {
stmt.setString(1, url)
stmt.addBatch()
}
stmt.executeBatch()
}
import(connection, keys)
}
} }
public suspend fun import(keys: Iterable<XteaKey>) { public suspend fun import(keys: Iterable<XteaKey>) {
database.execute { connection -> database.execute { connection ->
import(connection, keys)
}
}
private fun import(connection: Connection, keys: Iterable<XteaKey>) {
connection.prepareStatement( connection.prepareStatement(
""" """
LOCK TABLE keys IN EXCLUSIVE MODE LOCK TABLE keys IN EXCLUSIVE MODE
@ -104,4 +147,3 @@ public class KeyImporter @Inject constructor(
} }
} }
} }
}

@ -1,38 +1,19 @@
package org.openrs2.archive.key package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@Singleton @Singleton
public class OpenOsrsKeyDownloader @Inject constructor( public class OpenOsrsKeyDownloader @Inject constructor(
private val client: HttpClient, client: HttpClient,
private val jsonKeyReader: JsonKeyReader jsonKeyReader: JsonKeyReader
) : KeyDownloader { ) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun download(): Sequence<XteaKey> { override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val request = HttpRequest.newBuilder(ENDPOINT) return setOf(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
} }
private companion object { private companion object {
private val ENDPOINT = URI("https://xtea.openosrs.dev/get") private const val ENDPOINT = "https://xtea.openosrs.dev/get"
} }
} }

@ -0,0 +1,48 @@
package org.openrs2.archive.key
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext
import org.jsoup.Jsoup
import org.openrs2.http.charset
import org.openrs2.http.checkStatusCode
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import javax.inject.Inject
import javax.inject.Singleton
@Singleton
public class PolarKeyDownloader @Inject constructor(
private val client: HttpClient,
jsonKeyReader: JsonKeyReader
) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val request = HttpRequest.newBuilder(ENDPOINT)
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
val document = withContext(Dispatchers.IO) {
Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString())
}
val urls = mutableSetOf<String>()
for (element in document.select("a")) {
val url = element.absUrl("href")
if (url.endsWith(".json") && url !in seenUrls) {
urls += url
}
}
return urls
}
private companion object {
private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/")
}
}

@ -4,7 +4,6 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.future.await import kotlinx.coroutines.future.await
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import org.jdom2.input.SAXBuilder import org.jdom2.input.SAXBuilder
import org.openrs2.crypto.XteaKey
import org.openrs2.http.checkStatusCode import org.openrs2.http.checkStatusCode
import java.net.URI import java.net.URI
import java.net.http.HttpClient import java.net.http.HttpClient
@ -16,23 +15,11 @@ import javax.inject.Singleton
@Singleton @Singleton
public class RuneLiteKeyDownloader @Inject constructor( public class RuneLiteKeyDownloader @Inject constructor(
private val client: HttpClient, private val client: HttpClient,
private val jsonKeyReader: JsonKeyReader jsonKeyReader: JsonKeyReader
) : KeyDownloader { ) : JsonKeyDownloader(client, jsonKeyReader) {
override suspend fun download(): Sequence<XteaKey> { override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> {
val version = getVersion() val version = getVersion()
return setOf(getXteaEndpoint(version))
val request = HttpRequest.newBuilder(getXteaEndpoint(version))
.GET()
.build()
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await()
response.checkStatusCode()
return withContext(Dispatchers.IO) {
response.body().use { input ->
jsonKeyReader.read(input)
}
}
} }
private suspend fun getVersion(): String { private suspend fun getVersion(): String {
@ -58,8 +45,8 @@ public class RuneLiteKeyDownloader @Inject constructor(
private companion object { private companion object {
private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml") private val VERSION_ENDPOINT = URI("https://repo.runelite.net/net/runelite/runelite-parent/maven-metadata.xml")
private fun getXteaEndpoint(version: String): URI { private fun getXteaEndpoint(version: String): String {
return URI("https://api.runelite.net/runelite-$version/xtea") return "https://api.runelite.net/runelite-$version/xtea"
} }
} }
} }

@ -26,6 +26,10 @@ CREATE TABLE keys (
key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0) key xtea_key UNIQUE NOT NULL CHECK ((key).k0 <> 0 OR (key).k1 <> 0 OR (key).k2 <> 0 OR (key).k3 <> 0)
); );
CREATE TABLE keysets (
url TEXT PRIMARY KEY NOT NULL
);
CREATE TABLE containers ( CREATE TABLE containers (
id BIGSERIAL PRIMARY KEY NOT NULL, id BIGSERIAL PRIMARY KEY NOT NULL,
crc32 INTEGER NOT NULL, crc32 INTEGER NOT NULL,

@ -20,6 +20,7 @@ object Versions {
const val jdom = "2.0.6" const val jdom = "2.0.6"
const val jgrapht = "1.5.0" const val jgrapht = "1.5.0"
const val jimfs = "1.2" const val jimfs = "1.2"
const val jsoup = "1.13.1"
const val junit = "5.7.1" const val junit = "5.7.1"
const val kotlin = "1.4.30" const val kotlin = "1.4.30"
const val kotlinCoroutines = "1.4.2" const val kotlinCoroutines = "1.4.2"

Loading…
Cancel
Save