Add initial support for archiving clients

There are still some gaps but I want to get this committed and possibly
deployed before doing further work.

Remaining items include:

- Mach-O support
- New engine loader ArtifactLink support
- Post-668 client support
- FunOrb support

Signed-off-by: Graham <gpe@openrs2.org>
master
Graham 8 months ago
parent c65cc2ff59
commit 73e959a3cb
  1. 4
      archive/build.gradle.kts
  2. 2
      archive/src/main/kotlin/org/openrs2/archive/ArchiveCommand.kt
  3. 2
      archive/src/main/kotlin/org/openrs2/archive/ArchiveModule.kt
  4. 23
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  5. 11
      archive/src/main/kotlin/org/openrs2/archive/client/Architecture.kt
  6. 35
      archive/src/main/kotlin/org/openrs2/archive/client/Artifact.kt
  7. 46
      archive/src/main/kotlin/org/openrs2/archive/client/ArtifactFormat.kt
  8. 16
      archive/src/main/kotlin/org/openrs2/archive/client/ArtifactType.kt
  9. 14
      archive/src/main/kotlin/org/openrs2/archive/client/ClientCommand.kt
  10. 427
      archive/src/main/kotlin/org/openrs2/archive/client/ClientExporter.kt
  11. 740
      archive/src/main/kotlin/org/openrs2/archive/client/ClientImporter.kt
  12. 30
      archive/src/main/kotlin/org/openrs2/archive/client/ExportCommand.kt
  13. 25
      archive/src/main/kotlin/org/openrs2/archive/client/ImportCommand.kt
  14. 7
      archive/src/main/kotlin/org/openrs2/archive/client/Jvm.kt
  15. 43
      archive/src/main/kotlin/org/openrs2/archive/client/OperatingSystem.kt
  16. 16
      archive/src/main/kotlin/org/openrs2/archive/client/RefreshCommand.kt
  17. 82
      archive/src/main/kotlin/org/openrs2/archive/web/ClientsController.kt
  18. 4
      archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt
  19. 95
      archive/src/main/resources/org/openrs2/archive/migrations/V22__clients.sql
  20. 63
      archive/src/main/resources/org/openrs2/archive/templates/clients/index.html
  21. 129
      archive/src/main/resources/org/openrs2/archive/templates/clients/show.html
  22. 3
      archive/src/main/resources/org/openrs2/archive/templates/layout.html
  23. 9
      asm/src/main/kotlin/org/openrs2/asm/classpath/Library.kt
  24. 2
      gradle/libs.versions.toml

@ -12,6 +12,7 @@ dependencies {
api(libs.bundles.guice)
api(libs.clikt)
implementation(projects.asm)
implementation(projects.buffer)
implementation(projects.cache550)
implementation(projects.cli)
@ -30,15 +31,18 @@ dependencies {
implementation(libs.bundles.ktor)
implementation(libs.bundles.thymeleaf)
implementation(libs.byteUnits)
implementation(libs.cabParser)
implementation(libs.flyway)
implementation(libs.guava)
implementation(libs.hikaricp)
implementation(libs.jackson.jsr310)
implementation(libs.jdom)
implementation(libs.jelf)
implementation(libs.jquery)
implementation(libs.jsoup)
implementation(libs.kotlin.coroutines.core)
implementation(libs.netty.handler)
implementation(libs.pecoff4j)
implementation(libs.postgres)
}

@ -3,6 +3,7 @@ package org.openrs2.archive
import com.github.ajalt.clikt.core.NoOpCliktCommand
import com.github.ajalt.clikt.core.subcommands
import org.openrs2.archive.cache.CacheCommand
import org.openrs2.archive.client.ClientCommand
import org.openrs2.archive.key.KeyCommand
import org.openrs2.archive.name.NameCommand
import org.openrs2.archive.web.WebCommand
@ -13,6 +14,7 @@ public class ArchiveCommand : NoOpCliktCommand(name = "archive") {
init {
subcommands(
CacheCommand(),
ClientCommand(),
KeyCommand(),
NameCommand(),
WebCommand()

@ -10,6 +10,7 @@ import org.openrs2.archive.key.KeyDownloader
import org.openrs2.archive.key.RuneLiteKeyDownloader
import org.openrs2.archive.name.NameDownloader
import org.openrs2.archive.name.RuneStarNameDownloader
import org.openrs2.asm.AsmModule
import org.openrs2.buffer.BufferModule
import org.openrs2.cache.CacheModule
import org.openrs2.db.Database
@ -21,6 +22,7 @@ import javax.sql.DataSource
public object ArchiveModule : AbstractModule() {
override fun configure() {
install(AsmModule)
install(BufferModule)
install(CacheModule)
install(HttpModule)

@ -24,6 +24,8 @@ import org.openrs2.cache.StoreCorruptException
import org.openrs2.cache.VersionList
import org.openrs2.cache.VersionTrailer
import org.openrs2.crypto.Whirlpool
import org.openrs2.crypto.sha1
import org.openrs2.crypto.whirlpool
import org.openrs2.db.Database
import org.postgresql.util.PSQLState
import java.io.IOException
@ -84,7 +86,8 @@ public class CacheImporter @Inject constructor(
) : DefaultByteBufHolder(buf) {
public val bytes: ByteArray = ByteBufUtil.getBytes(buf, buf.readerIndex(), buf.readableBytes(), false)
public val crc32: Int = buf.crc32()
public val whirlpool: ByteArray = Whirlpool.whirlpool(bytes)
public val sha1: ByteArray = buf.sha1()
public val whirlpool: ByteArray = buf.whirlpool()
}
public class ChecksumTableBlob(
@ -854,6 +857,7 @@ public class CacheImporter @Inject constructor(
CREATE TEMPORARY TABLE tmp_blobs (
index INTEGER NOT NULL,
crc32 INTEGER NOT NULL,
sha1 BYTEA NOT NULL,
whirlpool BYTEA NOT NULL,
data BYTEA NOT NULL
) ON COMMIT DROP
@ -992,11 +996,11 @@ public class CacheImporter @Inject constructor(
return ids as List<Long>
}
private fun addBlob(connection: Connection, blob: Blob): Long {
public fun addBlob(connection: Connection, blob: Blob): Long {
return addBlobs(connection, listOf(blob)).single()
}
private fun addBlobs(connection: Connection, blobs: List<Blob>): List<Long> {
public fun addBlobs(connection: Connection, blobs: List<Blob>): List<Long> {
connection.prepareStatement(
"""
TRUNCATE TABLE tmp_blobs
@ -1007,15 +1011,16 @@ public class CacheImporter @Inject constructor(
connection.prepareStatement(
"""
INSERT INTO tmp_blobs (index, crc32, whirlpool, data)
VALUES (?, ?, ?, ?)
INSERT INTO tmp_blobs (index, crc32, sha1, whirlpool, data)
VALUES (?, ?, ?, ?, ?)
""".trimIndent()
).use { stmt ->
for ((i, blob) in blobs.withIndex()) {
stmt.setInt(1, i)
stmt.setInt(2, blob.crc32)
stmt.setBytes(3, blob.whirlpool)
stmt.setBytes(4, blob.bytes)
stmt.setBytes(3, blob.sha1)
stmt.setBytes(4, blob.whirlpool)
stmt.setBytes(5, blob.bytes)
stmt.addBatch()
}
@ -1025,8 +1030,8 @@ public class CacheImporter @Inject constructor(
connection.prepareStatement(
"""
INSERT INTO blobs (crc32, whirlpool, data)
SELECT t.crc32, t.whirlpool, t.data
INSERT INTO blobs (crc32, sha1, whirlpool, data)
SELECT t.crc32, t.sha1, t.whirlpool, t.data
FROM tmp_blobs t
LEFT JOIN blobs b ON b.whirlpool = t.whirlpool
WHERE b.whirlpool IS NULL

@ -0,0 +1,11 @@
package org.openrs2.archive.client
public enum class Architecture {
INDEPENDENT,
UNIVERSAL,
X86,
AMD64,
POWERPC,
SPARC,
SPARCV9
}

@ -0,0 +1,35 @@
package org.openrs2.archive.client
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufUtil
import org.openrs2.archive.cache.CacheExporter
import org.openrs2.archive.cache.CacheImporter
import java.time.Instant
public class Artifact(
data: ByteBuf,
public val game: String,
public val environment: String,
public val build: CacheExporter.Build?,
public val timestamp: Instant?,
public val type: ArtifactType,
public val format: ArtifactFormat,
public val os: OperatingSystem,
public val arch: Architecture,
public val jvm: Jvm,
public val links: List<ArtifactLink>
) : CacheImporter.Blob(data)
public data class ArtifactLink(
val type: ArtifactType,
val format: ArtifactFormat,
val os: OperatingSystem,
val arch: Architecture,
val jvm: Jvm,
val crc32: Int?,
val sha1: ByteArray,
val size: Int?
) {
public val sha1Hex: String
get() = ByteBufUtil.hexDump(sha1)
}

@ -0,0 +1,46 @@
package org.openrs2.archive.client
import io.ktor.http.ContentType
public enum class ArtifactFormat {
CAB,
JAR,
NATIVE,
PACK200,
PACKCLASS;
public fun getPrefix(os: OperatingSystem): String {
return when (this) {
NATIVE -> os.getPrefix()
else -> ""
}
}
public fun getExtension(os: OperatingSystem): String {
return when (this) {
CAB -> "cab"
JAR -> "jar"
NATIVE -> os.getExtension()
PACK200 -> "pack200"
PACKCLASS -> "js5"
}
}
public fun getContentType(os: OperatingSystem): ContentType {
return when (this) {
CAB -> CAB_MIME_TYPE
JAR -> JAR_MIME_TYPE
NATIVE -> os.getContentType()
PACK200, PACKCLASS -> ContentType.Application.OctetStream
}
}
public fun isJar(): Boolean {
return this != NATIVE
}
private companion object {
private val CAB_MIME_TYPE = ContentType("application", "vnd.ms-cab-compressed")
private val JAR_MIME_TYPE = ContentType("application", "java-archive")
}
}

@ -0,0 +1,16 @@
package org.openrs2.archive.client
public enum class ArtifactType {
BROWSERCONTROL,
CLIENT,
CLIENT_GL,
GLUEGEN_RT,
JAGGL,
JAGGL_DRI,
JAGMISC,
JOGL,
JOGL_AWT,
LOADER,
LOADER_GL,
UNPACKCLASS
}

@ -0,0 +1,14 @@
package org.openrs2.archive.client
import com.github.ajalt.clikt.core.NoOpCliktCommand
import com.github.ajalt.clikt.core.subcommands
public class ClientCommand : NoOpCliktCommand(name = "client") {
init {
subcommands(
ExportCommand(),
ImportCommand(),
RefreshCommand()
)
}
}

@ -0,0 +1,427 @@
package org.openrs2.archive.client
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufUtil
import io.netty.buffer.DefaultByteBufHolder
import io.netty.buffer.Unpooled
import jakarta.inject.Inject
import jakarta.inject.Singleton
import org.openrs2.archive.cache.CacheExporter
import org.openrs2.db.Database
import java.time.Instant
import java.time.ZoneOffset
import java.time.format.DateTimeFormatter
@Singleton
public class ClientExporter @Inject constructor(
private val database: Database
) {
public data class ArtifactSummary(
public val id: Long,
public val game: String,
public val environment: String,
public val build: CacheExporter.Build?,
public val timestamp: Instant?,
public val type: ArtifactType,
public val format: ArtifactFormat,
public val os: OperatingSystem,
public val arch: Architecture,
public val jvm: Jvm,
public val size: Int
) {
public val name: String
get() {
val builder = StringBuilder()
builder.append(format.getPrefix(os))
when (type) {
ArtifactType.CLIENT -> builder.append(game)
ArtifactType.CLIENT_GL -> builder.append("${game}_gl")
ArtifactType.GLUEGEN_RT -> builder.append("gluegen-rt")
else -> builder.append(type.name.lowercase())
}
if (jvm == Jvm.MICROSOFT) {
builder.append("ms")
}
if (os != OperatingSystem.INDEPENDENT) {
builder.append('-')
builder.append(os.name.lowercase())
}
if (arch != Architecture.INDEPENDENT) {
builder.append('-')
builder.append(arch.name.lowercase())
}
if (build != null) {
builder.append("-b")
builder.append(build)
}
if (timestamp != null) {
builder.append('-')
builder.append(
timestamp
.atOffset(ZoneOffset.UTC)
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss"))
)
}
builder.append("-openrs2#")
builder.append(id)
builder.append('.')
builder.append(format.getExtension(os))
return builder.toString()
}
}
public data class ArtifactLinkExport(
public val id: Long?,
public val build: CacheExporter.Build?,
public val timestamp: Instant?,
public val link: ArtifactLink
)
public class Artifact(
public val summary: ArtifactSummary,
public val crc32: Int,
public val sha1: ByteArray,
public val links: List<ArtifactLinkExport>
) {
public val sha1Hex: String
get() = ByteBufUtil.hexDump(sha1)
}
public class ArtifactExport(
public val summary: ArtifactSummary,
buf: ByteBuf
) : DefaultByteBufHolder(buf)
public suspend fun list(): List<ArtifactSummary> {
return database.execute { connection ->
connection.prepareStatement(
"""
SELECT
a.blob_id,
g.name,
e.name,
a.build_major,
a.build_minor,
a.timestamp,
a.type,
a.format,
a.os,
a.arch,
a.jvm,
length(b.data) AS size
FROM artifacts a
JOIN blobs b ON b.id = a.blob_id
JOIN games g ON g.id = a.game_id
JOIN environments e ON e.id = a.environment_id
ORDER BY a.build_major ASC, a.timestamp ASC, a.type ASC, a.format ASC, a.os ASC, a.arch ASC, a.jvm ASC
""".trimIndent()
).use { stmt ->
stmt.executeQuery().use { rows ->
val artifacts = mutableListOf<ArtifactSummary>()
while (rows.next()) {
val id = rows.getLong(1)
val game = rows.getString(2)
val environment = rows.getString(3)
var buildMajor: Int? = rows.getInt(4)
if (rows.wasNull()) {
buildMajor = null
}
var buildMinor: Int? = rows.getInt(5)
if (rows.wasNull()) {
buildMinor = null
}
val build = if (buildMajor != null) {
CacheExporter.Build(buildMajor, buildMinor)
} else {
null
}
val timestamp = rows.getTimestamp(6)?.toInstant()
val type = ArtifactType.valueOf(rows.getString(7).uppercase())
val format = ArtifactFormat.valueOf(rows.getString(8).uppercase())
val os = OperatingSystem.valueOf(rows.getString(9).uppercase())
val arch = Architecture.valueOf(rows.getString(10).uppercase())
val jvm = Jvm.valueOf(rows.getString(11).uppercase())
val size = rows.getInt(12)
artifacts += ArtifactSummary(
id,
game,
environment,
build,
timestamp,
type,
format,
os,
arch,
jvm,
size
)
}
return@execute artifacts
}
}
}
}
public suspend fun get(id: Long): Artifact? {
return database.execute { connection ->
val links = mutableListOf<ArtifactLinkExport>()
connection.prepareStatement(
"""
SELECT
a.blob_id,
a.build_major,
a.build_minor,
a.timestamp,
l.type,
l.format,
l.os,
l.arch,
l.jvm,
COALESCE(l.crc32, b.crc32),
l.sha1,
COALESCE(l.size, length(b.data))
FROM artifact_links l
LEFT JOIN blobs b ON b.sha1 = l.sha1
LEFT JOIN artifacts a ON a.blob_id = b.id
WHERE l.blob_id = ?
ORDER BY l.type, l.format, l.os, l.arch, l.jvm
""".trimIndent()
).use { stmt ->
stmt.setLong(1, id)
stmt.executeQuery().use { rows ->
while (rows.next()) {
var linkId: Long? = rows.getLong(1)
if (rows.wasNull()) {
linkId = null
}
var buildMajor: Int? = rows.getInt(2)
if (rows.wasNull()) {
buildMajor = null
}
var buildMinor: Int? = rows.getInt(3)
if (rows.wasNull()) {
buildMinor = null
}
val build = if (buildMajor != null) {
CacheExporter.Build(buildMajor, buildMinor)
} else {
null
}
val timestamp = rows.getTimestamp(4)?.toInstant()
val type = ArtifactType.valueOf(rows.getString(5).uppercase())
val format = ArtifactFormat.valueOf(rows.getString(6).uppercase())
val os = OperatingSystem.valueOf(rows.getString(7).uppercase())
val arch = Architecture.valueOf(rows.getString(8).uppercase())
val jvm = Jvm.valueOf(rows.getString(9).uppercase())
var crc32: Int? = rows.getInt(10)
if (rows.wasNull()) {
crc32 = null
}
val sha1 = rows.getBytes(11)
var size: Int? = rows.getInt(12)
if (rows.wasNull()) {
size = null
}
links += ArtifactLinkExport(
linkId,
build,
timestamp,
ArtifactLink(
type,
format,
os,
arch,
jvm,
crc32,
sha1,
size
)
)
}
}
}
connection.prepareStatement(
"""
SELECT
g.name,
e.name,
a.build_major,
a.build_minor,
a.timestamp,
a.type,
a.format,
a.os,
a.arch,
a.jvm,
length(b.data) AS size,
b.crc32,
b.sha1
FROM artifacts a
JOIN games g ON g.id = a.game_id
JOIN environments e ON e.id = a.environment_id
JOIN blobs b ON b.id = a.blob_id
WHERE a.blob_id = ?
""".trimIndent()
).use { stmt ->
stmt.setLong(1, id)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
return@execute null
}
val game = rows.getString(1)
val environment = rows.getString(2)
var buildMajor: Int? = rows.getInt(3)
if (rows.wasNull()) {
buildMajor = null
}
var buildMinor: Int? = rows.getInt(4)
if (rows.wasNull()) {
buildMinor = null
}
val build = if (buildMajor != null) {
CacheExporter.Build(buildMajor!!, buildMinor)
} else {
null
}
val timestamp = rows.getTimestamp(5)?.toInstant()
val type = ArtifactType.valueOf(rows.getString(6).uppercase())
val format = ArtifactFormat.valueOf(rows.getString(7).uppercase())
val os = OperatingSystem.valueOf(rows.getString(8).uppercase())
val arch = Architecture.valueOf(rows.getString(9).uppercase())
val jvm = Jvm.valueOf(rows.getString(10).uppercase())
val size = rows.getInt(11)
val crc32 = rows.getInt(12)
val sha1 = rows.getBytes(13)
return@execute Artifact(
ArtifactSummary(
id,
game,
environment,
build,
timestamp,
type,
format,
os,
arch,
jvm,
size
), crc32, sha1, links
)
}
}
}
}
public suspend fun export(id: Long): ArtifactExport? {
return database.execute { connection ->
connection.prepareStatement(
"""
SELECT
g.name,
e.name,
a.build_major,
a.build_minor,
a.timestamp,
a.type,
a.format,
a.os,
a.arch,
a.jvm,
b.data
FROM artifacts a
JOIN games g ON g.id = a.game_id
JOIN environments e ON e.id = a.environment_id
JOIN blobs b ON b.id = a.blob_id
WHERE a.blob_id = ?
""".trimIndent()
).use { stmt ->
stmt.setLong(1, id)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
return@execute null
}
val game = rows.getString(1)
val environment = rows.getString(2)
var buildMajor: Int? = rows.getInt(3)
if (rows.wasNull()) {
buildMajor = null
}
var buildMinor: Int? = rows.getInt(4)
if (rows.wasNull()) {
buildMinor = null
}
val build = if (buildMajor != null) {
CacheExporter.Build(buildMajor, buildMinor)
} else {
null
}
val timestamp = rows.getTimestamp(5)?.toInstant()
val type = ArtifactType.valueOf(rows.getString(6).uppercase())
val format = ArtifactFormat.valueOf(rows.getString(7).uppercase())
val os = OperatingSystem.valueOf(rows.getString(8).uppercase())
val arch = Architecture.valueOf(rows.getString(9).uppercase())
val jvm = Jvm.valueOf(rows.getString(10).uppercase())
val buf = Unpooled.wrappedBuffer(rows.getBytes(11))
val size = buf.readableBytes()
return@execute ArtifactExport(
ArtifactSummary(
id,
game,
environment,
build,
timestamp,
type,
format,
os,
arch,
jvm,
size
), buf
)
}
}
}
}
}

@ -0,0 +1,740 @@
package org.openrs2.archive.client
import com.github.michaelbull.logging.InlineLogger
import com.kichik.pecoff4j.PE
import com.kichik.pecoff4j.constant.MachineType
import com.kichik.pecoff4j.io.PEParser
import dorkbox.cabParser.CabParser
import dorkbox.cabParser.CabStreamSaver
import dorkbox.cabParser.structure.CabFileEntry
import io.netty.buffer.ByteBuf
import io.netty.buffer.ByteBufAllocator
import io.netty.buffer.ByteBufInputStream
import io.netty.buffer.ByteBufOutputStream
import io.netty.buffer.Unpooled
import io.netty.util.ByteProcessor
import jakarta.inject.Inject
import jakarta.inject.Singleton
import net.fornwall.jelf.ElfFile
import net.fornwall.jelf.ElfSymbol
import org.objectweb.asm.tree.ClassNode
import org.objectweb.asm.tree.LdcInsnNode
import org.objectweb.asm.tree.MethodInsnNode
import org.objectweb.asm.tree.TypeInsnNode
import org.openrs2.archive.cache.CacheExporter
import org.openrs2.archive.cache.CacheImporter
import org.openrs2.asm.InsnMatcher
import org.openrs2.asm.classpath.Library
import org.openrs2.asm.hasCode
import org.openrs2.asm.intConstant
import org.openrs2.asm.io.CabLibraryReader
import org.openrs2.asm.io.JarLibraryReader
import org.openrs2.asm.io.LibraryReader
import org.openrs2.asm.io.Pack200LibraryReader
import org.openrs2.asm.io.PackClassLibraryReader
import org.openrs2.buffer.use
import org.openrs2.compress.gzip.Gzip
import org.openrs2.db.Database
import org.openrs2.util.io.entries
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.InputStream
import java.io.OutputStream
import java.nio.file.Files
import java.nio.file.Path
import java.sql.Connection
import java.sql.Types
import java.time.Instant
import java.time.LocalDate
import java.time.Month
import java.time.ZoneOffset
import java.util.jar.JarInputStream
import java.util.jar.JarOutputStream
import java.util.jar.Pack200
@Singleton
public class ClientImporter @Inject constructor(
private val database: Database,
private val alloc: ByteBufAllocator,
private val packClassLibraryReader: PackClassLibraryReader,
private val importer: CacheImporter
) {
public suspend fun import(paths: Iterable<Path>) {
alloc.buffer().use { buf ->
for (path in paths) {
buf.clear()
Files.newInputStream(path).use { input ->
ByteBufOutputStream(buf).use { output ->
input.copyTo(output)
}
}
logger.info { "Importing $path" }
import(parse(buf))
}
}
}
public suspend fun import(artifact: Artifact) {
database.execute { connection ->
importer.prepare(connection)
import(connection, artifact)
}
}
private fun import(connection: Connection, artifact: Artifact) {
val id = importer.addBlob(connection, artifact)
val gameId = connection.prepareStatement(
"""
SELECT id
FROM games
WHERE name = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, artifact.game)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
throw IllegalArgumentException()
}
rows.getInt(1)
}
}
val environmentId = connection.prepareStatement(
"""
SELECT id
FROM environments
WHERE name = ?
""".trimIndent()
).use { stmt ->
stmt.setString(1, artifact.environment)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
throw IllegalArgumentException()
}
rows.getInt(1)
}
}
connection.prepareStatement(
"""
INSERT INTO artifacts (blob_id, game_id, environment_id, build_major, build_minor, timestamp, type, format, os, arch, jvm)
VALUES (?, ?, ?, ?, ?, ?, ?::artifact_type, ?::artifact_format, ?::os, ?::arch, ?::jvm)
ON CONFLICT (blob_id) DO UPDATE SET
game_id = EXCLUDED.game_id,
environment_id = EXCLUDED.environment_id,
build_major = EXCLUDED.build_major,
build_minor = EXCLUDED.build_minor,
timestamp = EXCLUDED.timestamp,
type = EXCLUDED.type,
format = EXCLUDED.format,
os = EXCLUDED.os,
arch = EXCLUDED.arch,
jvm = EXCLUDED.jvm
""".trimIndent()
).use { stmt ->
stmt.setLong(1, id)
stmt.setInt(2, gameId)
stmt.setInt(3, environmentId)
stmt.setObject(4, artifact.build?.major, Types.INTEGER)
stmt.setObject(5, artifact.build?.minor, Types.INTEGER)
stmt.setObject(6, artifact.timestamp?.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE)
stmt.setString(7, artifact.type.name.lowercase())
stmt.setString(8, artifact.format.name.lowercase())
stmt.setString(9, artifact.os.name.lowercase())
stmt.setString(10, artifact.arch.name.lowercase())
stmt.setString(11, artifact.jvm.name.lowercase())
stmt.execute()
}
connection.prepareStatement(
"""
DELETE FROM artifact_links
WHERE blob_id = ?
""".trimIndent()
).use { stmt ->
stmt.setLong(1, id)
stmt.execute()
}
connection.prepareStatement(
"""
INSERT INTO artifact_links (blob_id, type, format, os, arch, jvm, sha1, crc32, size)
VALUES (?, ?::artifact_type, ?::artifact_format, ?::os, ?::arch, ?::jvm, ?, ?, ?)
""".trimIndent()
).use { stmt ->
for (link in artifact.links) {
stmt.setLong(1, id)
stmt.setString(2, link.type.name.lowercase())
stmt.setString(3, link.format.name.lowercase())
stmt.setString(4, link.os.name.lowercase())
stmt.setString(5, link.arch.name.lowercase())
stmt.setString(6, link.jvm.name.lowercase())
stmt.setBytes(7, link.sha1)
stmt.setObject(8, link.crc32, Types.INTEGER)
stmt.setObject(9, link.size, Types.INTEGER)
stmt.addBatch()
}
stmt.executeBatch()
}
}
public suspend fun refresh() {
database.execute { connection ->
importer.prepare(connection)
var lastId: Long? = null
val blobs = mutableListOf<ByteArray>()
while (true) {
blobs.clear()
connection.prepareStatement(
"""
SELECT a.blob_id, b.data
FROM artifacts a
JOIN blobs b ON b.id = a.blob_id
WHERE ? IS NULL OR a.blob_id > ?
ORDER BY a.blob_id ASC
LIMIT 1024
""".trimIndent()
).use { stmt ->
stmt.setObject(1, lastId, Types.BIGINT)
stmt.setObject(2, lastId, Types.BIGINT)
stmt.executeQuery().use { rows ->
while (rows.next()) {
lastId = rows.getLong(1)
blobs += rows.getBytes(2)
}
}
}
if (blobs.isEmpty()) {
return@execute
}
for (blob in blobs) {
Unpooled.wrappedBuffer(blob).use { buf ->
import(connection, parse(buf))
}
}
}
}
}
private fun parse(buf: ByteBuf): Artifact {
return if (buf.hasPrefix(JAR)) {
parseJar(buf)
} else if (buf.hasPrefix(PACK200)) {
parsePack200(buf)
} else if (buf.hasPrefix(CAB)) {
parseCab(buf)
} else if (
buf.hasPrefix(PACKCLASS_UNCOMPRESSED) ||
buf.hasPrefix(PACKCLASS_BZIP2) ||
buf.hasPrefix(PACKCLASS_GZIP)
) {
parseLibrary(buf, packClassLibraryReader, ArtifactFormat.PACKCLASS)
} else if (buf.hasPrefix(ELF)) {
parseElf(buf)
} else if (buf.hasPrefix(PE)) {
parsePe(buf)
} else {
throw IllegalArgumentException()
}
}
private fun parseElf(buf: ByteBuf): Artifact {
val elf = ElfFile.from(ByteBufInputStream(buf.slice()))
val arch = when (elf.e_machine.toInt()) {
ElfFile.ARCH_i386 -> Architecture.X86
ElfFile.ARCH_X86_64 -> Architecture.AMD64
ElfFile.ARCH_SPARC -> Architecture.SPARC
ARCH_SPARCV9 -> Architecture.SPARCV9
else -> throw IllegalArgumentException()
}
val comment = String(elf.firstSectionByName(".comment").data)
val os = if (comment.contains(SOLARIS_COMMENT)) {
OperatingSystem.SOLARIS
} else {
OperatingSystem.LINUX
}
val symbols = elf.dynamicSymbolTableSection ?: throw IllegalArgumentException()
val type = getArtifactType(symbols.symbols.asSequence().mapNotNull(ElfSymbol::getName))
return Artifact(
buf.retain(),
"shared",
"live",
null,
null,
type,
ArtifactFormat.NATIVE,
os,
arch,
Jvm.SUN,
emptyList()
)
}
private fun getArtifactType(symbols: Sequence<String>): ArtifactType {
for (symbol in symbols) {
var name = symbol
if (name.startsWith('_')) {
name = name.substring(1)
}
if (name.startsWith("Java_")) { // RNI methods don't have a Java_ prefix
name = name.substring("Java_".length)
}
if (name.startsWith("jaggl_X11_dri_")) {
return ArtifactType.JAGGL_DRI
} else if (name.startsWith("jaggl_opengl_")) {
return ArtifactType.JAGGL
} else if (name.startsWith("com_sun_opengl_impl_GLImpl_")) {
return ArtifactType.JOGL
} else if (name.startsWith("com_sun_opengl_impl_JAWT_")) {
return ArtifactType.JOGL_AWT
} else if (name.startsWith("com_sun_gluegen_runtime_")) {
return ArtifactType.GLUEGEN_RT
} else if (name.startsWith("jagex3_jagmisc_jagmisc_")) {
return ArtifactType.JAGMISC
} else if (name.startsWith("nativeadvert_browsercontrol_")) {
return ArtifactType.BROWSERCONTROL
}
}
throw IllegalArgumentException()
}
private fun parsePe(buf: ByteBuf): Artifact {
val pe = PEParser.parse(ByteBufInputStream(buf.slice()))
val arch = when (pe.coffHeader.machine) {
MachineType.IMAGE_FILE_MACHINE_I386 -> Architecture.X86
MachineType.IMAGE_FILE_MACHINE_AMD64 -> Architecture.AMD64
else -> throw IllegalArgumentException()
}
val symbols = parsePeExportNames(buf, pe).toSet()
val type = getArtifactType(symbols.asSequence())
val jvm = if (symbols.contains("RNIGetCompatibleVersion")) {
Jvm.MICROSOFT
} else {
Jvm.SUN
}
return Artifact(
buf.retain(),
"shared",
"live",
null,
Instant.ofEpochSecond(pe.coffHeader.timeDateStamp.toLong()),
type,
ArtifactFormat.NATIVE,
OperatingSystem.WINDOWS,
arch,
jvm,
emptyList()
)
}
private fun parsePeExportNames(buf: ByteBuf, pe: PE): Sequence<String> {
return sequence {
val exportTable = pe.imageData.exportTable
val namePointerTable =
pe.sectionTable.rvaConverter.convertVirtualAddressToRawDataPointer(exportTable.namePointerRVA.toInt())
for (i in 0 until exportTable.numberOfNamePointers.toInt()) {
val namePointer = buf.readerIndex() + buf.getIntLE(buf.readerIndex() + namePointerTable + 4 * i)
val end = buf.forEachByte(namePointer, buf.writerIndex() - namePointer, ByteProcessor.FIND_NUL)
require(end != -1) {
"Unterminated string"
}
yield(buf.toString(namePointer, end - namePointer, Charsets.US_ASCII))
}
}
}
private fun parseJar(buf: ByteBuf): Artifact {
val timestamp = getJarTimestamp(ByteBufInputStream(buf.slice()))
return parseLibrary(buf, JarLibraryReader, ArtifactFormat.JAR, timestamp)
}
private fun parsePack200(buf: ByteBuf): Artifact {
val timestamp = ByteArrayOutputStream().use { tempOutput ->
Gzip.createHeaderlessInputStream(ByteBufInputStream(buf.slice())).use { gzipInput ->
JarOutputStream(tempOutput).use { jarOutput ->
Pack200.newUnpacker().unpack(gzipInput, jarOutput)
}
}
getJarTimestamp(ByteArrayInputStream(tempOutput.toByteArray()))
}
return parseLibrary(buf, Pack200LibraryReader, ArtifactFormat.PACK200, timestamp)
}
private fun parseCab(buf: ByteBuf): Artifact {
val timestamp = getCabTimestamp(ByteBufInputStream(buf.slice()))
return parseLibrary(buf, CabLibraryReader, ArtifactFormat.CAB, timestamp)
}
private fun getJarTimestamp(input: InputStream): Instant? {
var timestamp: Instant? = null
JarInputStream(input).use { jar ->
for (entry in jar.entries) {
val t = entry.lastModifiedTime?.toInstant()
if (timestamp == null || (t != null && t < timestamp)) {
timestamp = t
}
}
}
return timestamp
}
private fun getCabTimestamp(input: InputStream): Instant? {
var timestamp: Instant? = null
CabParser(input, object : CabStreamSaver {
override fun closeOutputStream(outputStream: OutputStream, entry: CabFileEntry) {
// entry
}
override fun openOutputStream(entry: CabFileEntry): OutputStream {
val t = entry.date.toInstant()
if (timestamp == null || t < timestamp) {
timestamp = t
}
return OutputStream.nullOutputStream()
}
override fun saveReservedAreaData(data: ByteArray?, dataLength: Int): Boolean {
return false
}
}).extractStream()
return timestamp
}
private fun parseLibrary(
buf: ByteBuf,
reader: LibraryReader,
format: ArtifactFormat,
timestamp: Instant? = null
): Artifact {
val library = Library.read("client", ByteBufInputStream(buf.slice()), reader)
val game: String
val build: CacheExporter.Build?
val type: ArtifactType
val links: List<ArtifactLink>
val mudclient = library["mudclient"]
val client = library["client"]
val loader = library["loader"]
if (mudclient != null) {
game = "classic"
build = null // TODO(gpe): classic support
type = ArtifactType.CLIENT
links = emptyList()
} else if (client != null) {
game = "runescape"
build = parseClientBuild(client)
type = if (build != null && build.major < COMBINED_BUILD && isClientGl(library)) {
ArtifactType.CLIENT_GL
} else {
ArtifactType.CLIENT
}
links = emptyList()
} else if (loader != null) {
if (isLoaderClassic(loader)) {
game = "classic"
build = null // TODO(gpe): classic support
type = ArtifactType.LOADER
links = emptyList() // TODO(gpe): classic support
} else {
game = "runescape"
build = parseLoaderBuild(library)
type = if (timestamp != null && timestamp < COMBINED_TIMESTAMP && isLoaderGl(library)) {
ArtifactType.LOADER_GL
} else {
ArtifactType.LOADER
}
links = parseLinks(library)
}
} else if (library.contains("mapview")) {
game = "mapview"
build = null
type = ArtifactType.CLIENT
links = emptyList()
} else if (library.contains("jaggl/opengl")) {
game = "shared"
type = ArtifactType.JAGGL
build = null
links = emptyList()
} else if (library.contains("com/sun/opengl/impl/GLImpl")) {
game = "shared"
type = ArtifactType.JOGL
build = null
links = emptyList()
} else if (library.contains("unpackclass")) {
game = "shared"
type = ArtifactType.UNPACKCLASS
build = null
links = emptyList()
} else {
throw IllegalArgumentException()
}
return Artifact(
buf.retain(),
game,
"live",
build,
timestamp,
type,
format,
OperatingSystem.INDEPENDENT,
Architecture.INDEPENDENT,
Jvm.INDEPENDENT,
links
)
}
private fun isClientGl(library: Library): Boolean {
for (clazz in library) {
for (method in clazz.methods) {
if (!method.hasCode) {
continue
}
for (insn in method.instructions) {
if (insn is MethodInsnNode && insn.name == "glBegin") {
return true
}
}
}
}
return false
}
private fun isLoaderClassic(clazz: ClassNode): Boolean {
for (method in clazz.methods) {
if (!method.hasCode) {
continue
}
for (insn in method.instructions) {
if (insn is LdcInsnNode && insn.cst == "mudclient") {
return true
}
}
}
return false
}
private fun isLoaderGl(library: Library): Boolean {
for (clazz in library) {
for (method in clazz.methods) {
if (!method.hasCode || method.name != "<clinit>") {
continue
}
for (insn in method.instructions) {
if (insn !is LdcInsnNode) {
continue
}
if (insn.cst == "jaggl.dll" || insn.cst == "jogl.dll") {
return true
}
}
}
}
return false
}
private fun parseClientBuild(clazz: ClassNode): CacheExporter.Build? {
for (method in clazz.methods) {
if (!method.hasCode || method.name != "main") {
continue
}
for (match in OLD_ENGINE_VERSION_MATCHER.match(method)) {
val ldc = match[0] as LdcInsnNode
if (ldc.cst != OLD_ENGINE_VERSION_STRING) {
continue
}
val version = match[2].intConstant
if (version != null) {
return CacheExporter.Build(version, null)
}
}
for (match in NEW_ENGINE_VERSION_MATCHER.match(method)) {
val new = match[0] as TypeInsnNode
if (new.desc != "client") {
continue
}
val candidates = mutableListOf<Int>()
for (insn in match) {
val candidate = insn.intConstant
if (candidate != null && candidate in NEW_ENGINE_BUILDS) {
candidates += candidate
}
}
candidates -= NEW_ENGINE_RESOLUTIONS
val version = candidates.singleOrNull()
if (version != null) {
return CacheExporter.Build(version, null)
}
}
}
return null
}
private fun parseLoaderBuild(library: Library): CacheExporter.Build? {
val clazz = library["sign/signlink"] ?: return null
for (field in clazz.fields) {
val value = field.value
if (field.name == "clientversion" && field.desc == "I" && value is Int) {
return CacheExporter.Build(value, null)
}
}
return null
}
private fun parseLinks(library: Library): List<ArtifactLink> {
val sig = library["sig"]
if (sig != null) {
var size: Int? = null
var sha1: ByteArray? = null
for (field in sig.fields) {
val value = field.value
if (field.name == "len" && field.desc == "I" && value is Int) {
size = value
}
}
for (method in sig.methods) {
if (!method.hasCode || method.name != "<clinit>") {
continue
}
for (match in SHA1_MATCHER.match(method)) {
val len = match[0].intConstant
if (len != SHA1_BYTES) {
continue
}
sha1 = ByteArray(SHA1_BYTES)
for (i in 2 until match.size step 4) {
val k = match[i + 1].intConstant!!
val v = match[i + 2].intConstant!!
sha1[k] = v.toByte()
}
}
}
require(size != null && sha1 != null)
return listOf(
ArtifactLink(