Compare commits
1 Commits
| Author | SHA1 | Date |
|---|---|---|
 Graham
|
2d19d3f90d | 2 years ago |
@ -0,0 +1,30 @@ |
|||||||
|
kind: pipeline |
||||||
|
type: docker |
||||||
|
name: default |
||||||
|
|
||||||
|
steps: |
||||||
|
- name: build |
||||||
|
image: registry.openrs2.org/openrs2-dev |
||||||
|
commands: |
||||||
|
- ./gradlew --no-daemon clean build |
||||||
|
|
||||||
|
- name: deploy |
||||||
|
image: registry.openrs2.org/openrs2-dev |
||||||
|
commands: |
||||||
|
- install -dm0700 $${HOME}/.ssh |
||||||
|
- echo -n "$${SSH_KEY}" > $${HOME}/.ssh/id_ed25519 |
||||||
|
- chmod 0600 $${HOME}/.ssh/id_ed25519 |
||||||
|
- ./gradlew --no-daemon publish |
||||||
|
environment: |
||||||
|
ORG_GRADLE_PROJECT_openrs2Username: |
||||||
|
from_secret: repo_username |
||||||
|
ORG_GRADLE_PROJECT_openrs2Password: |
||||||
|
from_secret: repo_password |
||||||
|
SSH_KEY: |
||||||
|
from_secret: ssh_key |
||||||
|
when: |
||||||
|
branch: |
||||||
|
- master |
||||||
|
event: |
||||||
|
exclude: |
||||||
|
- pull_request |
||||||
@ -1,34 +0,0 @@ |
|||||||
--- |
|
||||||
on: push |
|
||||||
jobs: |
|
||||||
build: |
|
||||||
runs-on: ubuntu-latest |
|
||||||
steps: |
|
||||||
- uses: actions/cache@v3 |
|
||||||
with: |
|
||||||
path: ~/.ssh/known_hosts |
|
||||||
key: ssh-known-hosts |
|
||||||
- uses: actions/checkout@v3 |
|
||||||
- uses: actions/setup-java@v3 |
|
||||||
with: |
|
||||||
distribution: temurin |
|
||||||
java-version: 11 |
|
||||||
- uses: gradle/wrapper-validation-action@v1 |
|
||||||
- uses: gradle/gradle-build-action@v2 |
|
||||||
with: |
|
||||||
arguments: build |
|
||||||
- if: github.ref == 'refs/heads/master' |
|
||||||
run: | |
|
||||||
install -dm0700 ~/.ssh |
|
||||||
touch ~/.ssh/id_ed25519 |
|
||||||
chmod 0600 ~/.ssh/id_ed25519 |
|
||||||
echo "${SSH_KEY}" > ~/.ssh/id_ed25519 |
|
||||||
env: |
|
||||||
SSH_KEY: ${{ secrets.SSH_KEY }} |
|
||||||
- if: github.ref == 'refs/heads/master' |
|
||||||
uses: gradle/gradle-build-action@v2 |
|
||||||
with: |
|
||||||
arguments: publish |
|
||||||
env: |
|
||||||
ORG_GRADLE_PROJECT_openrs2Username: ${{ secrets.REPO_USERNAME }} |
|
||||||
ORG_GRADLE_PROJECT_openrs2Password: ${{ secrets.REPO_PASSWORD }} |
|
||||||
@ -1,16 +0,0 @@ |
|||||||
package org.openrs2.archive.cache |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.CliktCommand |
|
||||||
import com.google.inject.Guice |
|
||||||
import kotlinx.coroutines.runBlocking |
|
||||||
import org.openrs2.archive.ArchiveModule |
|
||||||
import org.openrs2.inject.CloseableInjector |
|
||||||
|
|
||||||
public class CrossPollinateCommand : CliktCommand(name = "cross-pollinate") { |
|
||||||
override fun run(): Unit = runBlocking { |
|
||||||
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> |
|
||||||
val crossPollinator = injector.getInstance(CrossPollinator::class.java) |
|
||||||
crossPollinator.crossPollinate() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,223 +0,0 @@ |
|||||||
package org.openrs2.archive.cache |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import io.netty.buffer.ByteBufAllocator |
|
||||||
import io.netty.buffer.ByteBufInputStream |
|
||||||
import io.netty.buffer.Unpooled |
|
||||||
import jakarta.inject.Inject |
|
||||||
import jakarta.inject.Singleton |
|
||||||
import org.openrs2.buffer.crc32 |
|
||||||
import org.openrs2.buffer.use |
|
||||||
import org.openrs2.cache.Js5Compression |
|
||||||
import org.openrs2.cache.Js5CompressionType |
|
||||||
import org.openrs2.db.Database |
|
||||||
import java.sql.Connection |
|
||||||
import java.util.zip.GZIPInputStream |
|
||||||
|
|
||||||
@Singleton |
|
||||||
public class CrossPollinator @Inject constructor( |
|
||||||
private val database: Database, |
|
||||||
private val alloc: ByteBufAllocator, |
|
||||||
private val importer: CacheImporter |
|
||||||
) { |
|
||||||
public suspend fun crossPollinate() { |
|
||||||
database.execute { connection -> |
|
||||||
for ((index, archive) in OLD_TO_NEW_ENGINE) { |
|
||||||
crossPollinate(connection, index, archive) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun crossPollinate(connection: Connection, index: Int, archive: Int) { |
|
||||||
val scopeId: Int |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT id |
|
||||||
FROM scopes |
|
||||||
WHERE name = 'runescape' |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
check(rows.next()) |
|
||||||
|
|
||||||
scopeId = rows.getInt(1) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
val groups = mutableListOf<CacheImporter.Group>() |
|
||||||
val files = mutableListOf<CacheImporter.File>() |
|
||||||
|
|
||||||
try { |
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT |
|
||||||
new.group_id AS id, |
|
||||||
old.version AS old_version, |
|
||||||
old.crc32 AS old_crc32, |
|
||||||
b.data AS old_data, |
|
||||||
new.version AS new_version, |
|
||||||
new.crc32 AS new_crc32, |
|
||||||
c.data AS new_data |
|
||||||
FROM ( |
|
||||||
SELECT DISTINCT vf.index_id, vf.file_id, vf.version, vf.crc32 |
|
||||||
FROM version_list_files vf |
|
||||||
WHERE vf.blob_id IN ( |
|
||||||
SELECT v.blob_id |
|
||||||
FROM version_lists v |
|
||||||
JOIN resolved_archives a ON a.blob_id = v.blob_id AND a.archive_id = 5 |
|
||||||
) AND vf.index_id = ? |
|
||||||
) old |
|
||||||
JOIN ( |
|
||||||
SELECT DISTINCT ig.group_id, ig.version, ig.crc32 |
|
||||||
FROM index_groups ig |
|
||||||
WHERE ig.container_id IN ( |
|
||||||
SELECT i.container_id |
|
||||||
FROM resolved_indexes i |
|
||||||
WHERE i.scope_id = ? AND i.archive_id = ? |
|
||||||
) |
|
||||||
) new ON old.file_id = new.group_id AND old.version = new.version + 1 |
|
||||||
LEFT JOIN resolve_file(old.index_id, old.file_id, old.version, old.crc32) b ON TRUE |
|
||||||
LEFT JOIN resolve_group(?, ?::uint1, new.group_id, new.crc32, new.version) c ON TRUE |
|
||||||
WHERE (b.data IS NULL AND c.data IS NOT NULL) OR (b.data IS NOT NULL AND c.data IS NULL) |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setInt(1, index) |
|
||||||
stmt.setInt(2, scopeId) |
|
||||||
stmt.setInt(3, archive) |
|
||||||
stmt.setInt(4, scopeId) |
|
||||||
stmt.setInt(5, archive) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
while (rows.next()) { |
|
||||||
val id = rows.getInt(1) |
|
||||||
val oldVersion = rows.getInt(2) |
|
||||||
val oldChecksum = rows.getInt(3) |
|
||||||
val newVersion = rows.getInt(5) |
|
||||||
val newChecksum = rows.getInt(6) |
|
||||||
|
|
||||||
val oldData = rows.getBytes(4) |
|
||||||
if (oldData != null) { |
|
||||||
Unpooled.wrappedBuffer(oldData).use { oldBuf -> |
|
||||||
fileToGroup(oldBuf, newChecksum).use { newBuf -> |
|
||||||
if (newBuf != null) { |
|
||||||
val uncompressed = Js5Compression.uncompressUnlessEncrypted(newBuf.slice()) |
|
||||||
groups += CacheImporter.Group( |
|
||||||
archive, |
|
||||||
id, |
|
||||||
newBuf.retain(), |
|
||||||
uncompressed, |
|
||||||
newVersion, |
|
||||||
false |
|
||||||
) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
val newData = rows.getBytes(7) |
|
||||||
if (newData != null) { |
|
||||||
Unpooled.wrappedBuffer(newData).use { newBuf -> |
|
||||||
val oldBuf = groupToFile(newBuf, oldChecksum) |
|
||||||
if (oldBuf != null) { |
|
||||||
files += CacheImporter.File(index, id, oldBuf, oldVersion) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
if (groups.isEmpty() && files.isEmpty()) { |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
importer.prepare(connection) |
|
||||||
|
|
||||||
val sourceId = importer.addSource( |
|
||||||
connection, |
|
||||||
type = CacheImporter.SourceType.CROSS_POLLINATION, |
|
||||||
cacheId = null, |
|
||||||
gameId = null, |
|
||||||
buildMajor = null, |
|
||||||
buildMinor = null, |
|
||||||
timestamp = null, |
|
||||||
name = null, |
|
||||||
description = null, |
|
||||||
url = null, |
|
||||||
) |
|
||||||
|
|
||||||
if (groups.isNotEmpty()) { |
|
||||||
importer.addGroups(connection, scopeId, sourceId, groups) |
|
||||||
} |
|
||||||
|
|
||||||
if (files.isNotEmpty()) { |
|
||||||
importer.addFiles(connection, sourceId, files) |
|
||||||
} |
|
||||||
} finally { |
|
||||||
groups.forEach(CacheImporter.Group::release) |
|
||||||
files.forEach(CacheImporter.File::release) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun getUncompressedLength(buf: ByteBuf): Int { |
|
||||||
GZIPInputStream(ByteBufInputStream(buf)).use { input -> |
|
||||||
var len = 0 |
|
||||||
val temp = ByteArray(4096) |
|
||||||
|
|
||||||
while (true) { |
|
||||||
val n = input.read(temp) |
|
||||||
if (n == -1) { |
|
||||||
break |
|
||||||
} |
|
||||||
len += n |
|
||||||
} |
|
||||||
|
|
||||||
return len |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun fileToGroup(input: ByteBuf, expectedChecksum: Int): ByteBuf? { |
|
||||||
val len = input.readableBytes() |
|
||||||
val lenWithHeader = len + JS5_COMPRESSION_HEADER_LEN |
|
||||||
val uncompressedLen = getUncompressedLength(input.slice()) |
|
||||||
|
|
||||||
alloc.buffer(lenWithHeader, lenWithHeader).use { output -> |
|
||||||
output.writeByte(Js5CompressionType.GZIP.ordinal) |
|
||||||
output.writeInt(len) |
|
||||||
output.writeInt(uncompressedLen) |
|
||||||
output.writeBytes(input) |
|
||||||
|
|
||||||
return if (output.crc32() == expectedChecksum) { |
|
||||||
output.retain() |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun groupToFile(input: ByteBuf, expectedChecksum: Int): ByteBuf? { |
|
||||||
val type = Js5CompressionType.fromOrdinal(input.readUnsignedByte().toInt()) |
|
||||||
if (type != Js5CompressionType.GZIP) { |
|
||||||
return null |
|
||||||
} |
|
||||||
|
|
||||||
input.skipBytes(JS5_COMPRESSION_HEADER_LEN - 1) |
|
||||||
|
|
||||||
return if (input.crc32() == expectedChecksum) { |
|
||||||
input.retainedSlice() |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private val OLD_TO_NEW_ENGINE = mapOf( |
|
||||||
1 to 7, // MODELS |
|
||||||
3 to 6, // MIDI_SONGS |
|
||||||
4 to 5, // MAPS |
|
||||||
) |
|
||||||
|
|
||||||
private const val JS5_COMPRESSION_HEADER_LEN = 9 |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,149 +0,0 @@ |
|||||||
package org.openrs2.archive.cache.finder |
|
||||||
|
|
||||||
import com.github.michaelbull.logging.InlineLogger |
|
||||||
import com.google.common.io.ByteStreams |
|
||||||
import com.google.common.io.LittleEndianDataInputStream |
|
||||||
import org.openrs2.util.charset.Cp1252Charset |
|
||||||
import java.io.Closeable |
|
||||||
import java.io.EOFException |
|
||||||
import java.io.IOException |
|
||||||
import java.io.InputStream |
|
||||||
import java.io.PushbackInputStream |
|
||||||
import java.nio.file.Files |
|
||||||
import java.nio.file.Path |
|
||||||
import java.nio.file.attribute.BasicFileAttributeView |
|
||||||
import java.nio.file.attribute.FileTime |
|
||||||
import java.time.Instant |
|
||||||
|
|
||||||
public class CacheFinderExtractor( |
|
||||||
input: InputStream |
|
||||||
) : Closeable { |
|
||||||
private val pushbackInput = PushbackInputStream(input) |
|
||||||
private val input = LittleEndianDataInputStream(pushbackInput) |
|
||||||
|
|
||||||
private fun readTimestamp(): FileTime { |
|
||||||
val lo = input.readInt().toLong() and 0xFFFFFFFF |
|
||||||
val hi = input.readInt().toLong() and 0xFFFFFFFF |
|
||||||
|
|
||||||
val seconds = (((hi shl 32) or lo) / 10_000_000) - FILETIME_TO_UNIX_EPOCH |
|
||||||
|
|
||||||
return FileTime.from(Instant.ofEpochSecond(seconds, lo)) |
|
||||||
} |
|
||||||
|
|
||||||
private fun readName(): String { |
|
||||||
val bytes = ByteArray(MAX_PATH) |
|
||||||
input.readFully(bytes) |
|
||||||
|
|
||||||
var len = bytes.size |
|
||||||
for ((i, b) in bytes.withIndex()) { |
|
||||||
if (b.toInt() == 0) { |
|
||||||
len = i |
|
||||||
break |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return String(bytes, 0, len, Cp1252Charset) |
|
||||||
} |
|
||||||
|
|
||||||
private fun peekUnsignedByte(): Int { |
|
||||||
val n = pushbackInput.read() |
|
||||||
pushbackInput.unread(n) |
|
||||||
return n |
|
||||||
} |
|
||||||
|
|
||||||
public fun extract(destination: Path) { |
|
||||||
val newVersion = peekUnsignedByte() == 0xFE |
|
||||||
if (newVersion) { |
|
||||||
val signature = input.readInt() |
|
||||||
if (signature != 0x435352FE) { |
|
||||||
throw IOException("Invalid signature") |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
var readDirectoryPath = true |
|
||||||
var number = 0 |
|
||||||
var directorySuffix: String? = null |
|
||||||
|
|
||||||
while (true) { |
|
||||||
if (newVersion && readDirectoryPath) { |
|
||||||
val len = try { |
|
||||||
input.readInt() |
|
||||||
} catch (ex: EOFException) { |
|
||||||
break |
|
||||||
} |
|
||||||
|
|
||||||
val bytes = ByteArray(len) |
|
||||||
input.readFully(bytes) |
|
||||||
|
|
||||||
val path = String(bytes, Cp1252Charset) |
|
||||||
logger.info { "Extracting $path" } |
|
||||||
|
|
||||||
readDirectoryPath = false |
|
||||||
directorySuffix = path.substring(path.lastIndexOf('\\') + 1) |
|
||||||
.replace(INVALID_CHARS, "_") |
|
||||||
|
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
if (peekUnsignedByte() == 0xFF) { |
|
||||||
input.skipBytes(1) |
|
||||||
readDirectoryPath = true |
|
||||||
number++ |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
val attributes = try { |
|
||||||
input.readInt() |
|
||||||
} catch (ex: EOFException) { |
|
||||||
break |
|
||||||
} |
|
||||||
|
|
||||||
val btime = readTimestamp() |
|
||||||
val atime = readTimestamp() |
|
||||||
val mtime = readTimestamp() |
|
||||||
|
|
||||||
val sizeHi = input.readInt().toLong() and 0xFFFFFFFF |
|
||||||
val sizeLo = input.readInt().toLong() and 0xFFFFFFFF |
|
||||||
val size = (sizeHi shl 32) or sizeLo |
|
||||||
|
|
||||||
input.skipBytes(8) // reserved |
|
||||||
|
|
||||||
val name = readName() |
|
||||||
|
|
||||||
input.skipBytes(14) // alternate name |
|
||||||
input.skipBytes(2) // padding |
|
||||||
|
|
||||||
val dir = if (directorySuffix != null) { |
|
||||||
destination.resolve("cache${number}_$directorySuffix") |
|
||||||
} else { |
|
||||||
destination.resolve("cache$number") |
|
||||||
} |
|
||||||
|
|
||||||
Files.createDirectories(dir) |
|
||||||
|
|
||||||
if ((attributes and FILE_ATTRIBUTE_DIRECTORY) == 0) { |
|
||||||
val file = dir.resolve(name) |
|
||||||
|
|
||||||
Files.newOutputStream(file).use { output -> |
|
||||||
ByteStreams.copy(ByteStreams.limit(input, size), output) |
|
||||||
} |
|
||||||
|
|
||||||
val view = Files.getFileAttributeView(file, BasicFileAttributeView::class.java) |
|
||||||
view.setTimes(mtime, atime, btime) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
override fun close() { |
|
||||||
input.close() |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private const val FILETIME_TO_UNIX_EPOCH: Long = 11644473600 |
|
||||||
private const val MAX_PATH = 260 |
|
||||||
private const val FILE_ATTRIBUTE_DIRECTORY = 0x10 |
|
||||||
private val INVALID_CHARS = Regex("[^A-Za-z0-9-]") |
|
||||||
|
|
||||||
private val logger = InlineLogger() |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,25 +0,0 @@ |
|||||||
package org.openrs2.archive.cache.finder |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.CliktCommand |
|
||||||
import com.github.ajalt.clikt.parameters.arguments.argument |
|
||||||
import com.github.ajalt.clikt.parameters.arguments.default |
|
||||||
import com.github.ajalt.clikt.parameters.types.inputStream |
|
||||||
import com.github.ajalt.clikt.parameters.types.path |
|
||||||
import java.nio.file.Path |
|
||||||
|
|
||||||
public class ExtractCommand : CliktCommand(name = "extract") { |
|
||||||
private val input by argument().inputStream() |
|
||||||
private val output by argument().path( |
|
||||||
mustExist = false, |
|
||||||
canBeFile = false, |
|
||||||
canBeDir = true, |
|
||||||
mustBeReadable = true, |
|
||||||
mustBeWritable = true |
|
||||||
).default(Path.of(".")) |
|
||||||
|
|
||||||
override fun run() { |
|
||||||
CacheFinderExtractor(input).use { extractor -> |
|
||||||
extractor.extract(output) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,8 +1,25 @@ |
|||||||
package org.openrs2.archive.cache.nxt |
package org.openrs2.archive.cache.nxt |
||||||
|
|
||||||
import org.openrs2.protocol.EmptyPacketCodec |
import io.netty.buffer.ByteBuf |
||||||
|
import org.openrs2.crypto.StreamCipher |
||||||
|
import org.openrs2.protocol.PacketCodec |
||||||
|
|
||||||
public object Js5OkCodec : EmptyPacketCodec<LoginResponse.Js5Ok>( |
public object Js5OkCodec : PacketCodec<LoginResponse.Js5Ok>( |
||||||
opcode = 0, |
opcode = 0, |
||||||
packet = LoginResponse.Js5Ok |
length = LoginResponse.Js5Ok.LOADING_REQUIREMENTS * 4, |
||||||
) |
type = LoginResponse.Js5Ok::class.java |
||||||
|
) { |
||||||
|
override fun decode(input: ByteBuf, cipher: StreamCipher): LoginResponse.Js5Ok { |
||||||
|
val loadingRequirements = mutableListOf<Int>() |
||||||
|
for (i in 0 until LoginResponse.Js5Ok.LOADING_REQUIREMENTS) { |
||||||
|
loadingRequirements += input.readInt() |
||||||
|
} |
||||||
|
return LoginResponse.Js5Ok(loadingRequirements) |
||||||
|
} |
||||||
|
|
||||||
|
override fun encode(input: LoginResponse.Js5Ok, output: ByteBuf, cipher: StreamCipher) { |
||||||
|
for (requirement in input.loadingRequirements) { |
||||||
|
output.writeInt(requirement) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|||||||
@ -1,11 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
public enum class Architecture { |
|
||||||
INDEPENDENT, |
|
||||||
UNIVERSAL, |
|
||||||
X86, |
|
||||||
AMD64, |
|
||||||
POWERPC, |
|
||||||
SPARC, |
|
||||||
SPARCV9 |
|
||||||
} |
|
||||||
@ -1,35 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import io.netty.buffer.ByteBufUtil |
|
||||||
import org.openrs2.archive.cache.CacheExporter |
|
||||||
import org.openrs2.archive.cache.CacheImporter |
|
||||||
import java.time.Instant |
|
||||||
|
|
||||||
public class Artifact( |
|
||||||
data: ByteBuf, |
|
||||||
public val game: String, |
|
||||||
public val environment: String, |
|
||||||
public val build: CacheExporter.Build?, |
|
||||||
public val timestamp: Instant?, |
|
||||||
public val type: ArtifactType, |
|
||||||
public val format: ArtifactFormat, |
|
||||||
public val os: OperatingSystem, |
|
||||||
public val arch: Architecture, |
|
||||||
public val jvm: Jvm, |
|
||||||
public val links: List<ArtifactLink> |
|
||||||
) : CacheImporter.Blob(data) |
|
||||||
|
|
||||||
public data class ArtifactLink( |
|
||||||
val type: ArtifactType, |
|
||||||
val format: ArtifactFormat, |
|
||||||
val os: OperatingSystem, |
|
||||||
val arch: Architecture, |
|
||||||
val jvm: Jvm, |
|
||||||
val crc32: Int?, |
|
||||||
val sha1: ByteArray, |
|
||||||
val size: Int? |
|
||||||
) { |
|
||||||
public val sha1Hex: String |
|
||||||
get() = ByteBufUtil.hexDump(sha1) |
|
||||||
} |
|
||||||
@ -1,46 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import io.ktor.http.ContentType |
|
||||||
|
|
||||||
public enum class ArtifactFormat { |
|
||||||
CAB, |
|
||||||
JAR, |
|
||||||
NATIVE, |
|
||||||
PACK200, |
|
||||||
PACKCLASS; |
|
||||||
|
|
||||||
public fun getPrefix(os: OperatingSystem): String { |
|
||||||
return when (this) { |
|
||||||
NATIVE -> os.getPrefix() |
|
||||||
else -> "" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public fun getExtension(os: OperatingSystem): String { |
|
||||||
return when (this) { |
|
||||||
CAB -> "cab" |
|
||||||
JAR -> "jar" |
|
||||||
NATIVE -> os.getExtension() |
|
||||||
PACK200 -> "pack200" |
|
||||||
PACKCLASS -> "js5" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public fun getContentType(os: OperatingSystem): ContentType { |
|
||||||
return when (this) { |
|
||||||
CAB -> CAB_MIME_TYPE |
|
||||||
JAR -> JAR_MIME_TYPE |
|
||||||
NATIVE -> os.getContentType() |
|
||||||
PACK200, PACKCLASS -> ContentType.Application.OctetStream |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public fun isJar(): Boolean { |
|
||||||
return this != NATIVE |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private val CAB_MIME_TYPE = ContentType("application", "vnd.ms-cab-compressed") |
|
||||||
private val JAR_MIME_TYPE = ContentType("application", "java-archive") |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,16 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
public enum class ArtifactType { |
|
||||||
BROWSERCONTROL, |
|
||||||
CLIENT, |
|
||||||
CLIENT_GL, |
|
||||||
GLUEGEN_RT, |
|
||||||
JAGGL, |
|
||||||
JAGGL_DRI, |
|
||||||
JAGMISC, |
|
||||||
JOGL, |
|
||||||
JOGL_AWT, |
|
||||||
LOADER, |
|
||||||
LOADER_GL, |
|
||||||
UNPACKCLASS |
|
||||||
} |
|
||||||
@ -1,14 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.NoOpCliktCommand |
|
||||||
import com.github.ajalt.clikt.core.subcommands |
|
||||||
|
|
||||||
public class ClientCommand : NoOpCliktCommand(name = "client") { |
|
||||||
init { |
|
||||||
subcommands( |
|
||||||
ExportCommand(), |
|
||||||
ImportCommand(), |
|
||||||
RefreshCommand() |
|
||||||
) |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,455 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import io.netty.buffer.ByteBufUtil |
|
||||||
import io.netty.buffer.DefaultByteBufHolder |
|
||||||
import io.netty.buffer.Unpooled |
|
||||||
import jakarta.inject.Inject |
|
||||||
import jakarta.inject.Singleton |
|
||||||
import org.openrs2.archive.cache.CacheExporter |
|
||||||
import org.openrs2.db.Database |
|
||||||
import java.time.Instant |
|
||||||
import java.time.ZoneOffset |
|
||||||
import java.time.format.DateTimeFormatter |
|
||||||
|
|
||||||
@Singleton |
|
||||||
public class ClientExporter @Inject constructor( |
|
||||||
private val database: Database |
|
||||||
) { |
|
||||||
public data class ArtifactSummary( |
|
||||||
public val id: Long, |
|
||||||
public val game: String, |
|
||||||
public val environment: String, |
|
||||||
public val build: CacheExporter.Build?, |
|
||||||
public val timestamp: Instant?, |
|
||||||
public val type: ArtifactType, |
|
||||||
public val format: ArtifactFormat, |
|
||||||
public val os: OperatingSystem, |
|
||||||
public val arch: Architecture, |
|
||||||
public val jvm: Jvm, |
|
||||||
public val size: Int |
|
||||||
) { |
|
||||||
public val name: String |
|
||||||
get() { |
|
||||||
val builder = StringBuilder() |
|
||||||
builder.append(format.getPrefix(os)) |
|
||||||
|
|
||||||
when (type) { |
|
||||||
ArtifactType.CLIENT -> builder.append(game) |
|
||||||
ArtifactType.CLIENT_GL -> builder.append("${game}_gl") |
|
||||||
ArtifactType.GLUEGEN_RT -> builder.append("gluegen-rt") |
|
||||||
else -> builder.append(type.name.lowercase()) |
|
||||||
} |
|
||||||
|
|
||||||
if (jvm == Jvm.MICROSOFT) { |
|
||||||
builder.append("ms") |
|
||||||
} |
|
||||||
|
|
||||||
if (os != OperatingSystem.INDEPENDENT) { |
|
||||||
builder.append('-') |
|
||||||
builder.append(os.name.lowercase()) |
|
||||||
} |
|
||||||
|
|
||||||
if (arch != Architecture.INDEPENDENT) { |
|
||||||
builder.append('-') |
|
||||||
builder.append(arch.name.lowercase()) |
|
||||||
} |
|
||||||
|
|
||||||
if (build != null) { |
|
||||||
builder.append("-b") |
|
||||||
builder.append(build) |
|
||||||
} |
|
||||||
|
|
||||||
if (timestamp != null) { |
|
||||||
builder.append('-') |
|
||||||
builder.append( |
|
||||||
timestamp |
|
||||||
.atOffset(ZoneOffset.UTC) |
|
||||||
.format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss")) |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
builder.append("-openrs2#") |
|
||||||
builder.append(id) |
|
||||||
|
|
||||||
builder.append('.') |
|
||||||
builder.append(format.getExtension(os)) |
|
||||||
|
|
||||||
return builder.toString() |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public data class ArtifactSource( |
|
||||||
public val name: String?, |
|
||||||
public val description: String?, |
|
||||||
public val url: String? |
|
||||||
) |
|
||||||
|
|
||||||
public data class ArtifactLinkExport( |
|
||||||
public val id: Long?, |
|
||||||
public val build: CacheExporter.Build?, |
|
||||||
public val timestamp: Instant?, |
|
||||||
public val link: ArtifactLink |
|
||||||
) |
|
||||||
|
|
||||||
public class Artifact( |
|
||||||
public val summary: ArtifactSummary, |
|
||||||
public val crc32: Int, |
|
||||||
public val sha1: ByteArray, |
|
||||||
public val sources: List<ArtifactSource>, |
|
||||||
public val links: List<ArtifactLinkExport> |
|
||||||
) { |
|
||||||
public val sha1Hex: String |
|
||||||
get() = ByteBufUtil.hexDump(sha1) |
|
||||||
} |
|
||||||
|
|
||||||
public class ArtifactExport( |
|
||||||
public val summary: ArtifactSummary, |
|
||||||
buf: ByteBuf |
|
||||||
) : DefaultByteBufHolder(buf) |
|
||||||
|
|
||||||
public suspend fun list(): List<ArtifactSummary> { |
|
||||||
return database.execute { connection -> |
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT |
|
||||||
a.blob_id, |
|
||||||
g.name, |
|
||||||
e.name, |
|
||||||
a.build_major, |
|
||||||
a.build_minor, |
|
||||||
a.timestamp, |
|
||||||
a.type, |
|
||||||
a.format, |
|
||||||
a.os, |
|
||||||
a.arch, |
|
||||||
a.jvm, |
|
||||||
length(b.data) AS size |
|
||||||
FROM artifacts a |
|
||||||
JOIN blobs b ON b.id = a.blob_id |
|
||||||
JOIN games g ON g.id = a.game_id |
|
||||||
JOIN environments e ON e.id = a.environment_id |
|
||||||
ORDER BY a.build_major ASC, a.timestamp ASC, a.type ASC, a.format ASC, a.os ASC, a.arch ASC, a.jvm ASC |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
val artifacts = mutableListOf<ArtifactSummary>() |
|
||||||
|
|
||||||
while (rows.next()) { |
|
||||||
val id = rows.getLong(1) |
|
||||||
val game = rows.getString(2) |
|
||||||
val environment = rows.getString(3) |
|
||||||
|
|
||||||
var buildMajor: Int? = rows.getInt(4) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMajor = null |
|
||||||
} |
|
||||||
|
|
||||||
var buildMinor: Int? = rows.getInt(5) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMinor = null |
|
||||||
} |
|
||||||
|
|
||||||
val build = if (buildMajor != null) { |
|
||||||
CacheExporter.Build(buildMajor, buildMinor) |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
|
|
||||||
val timestamp = rows.getTimestamp(6)?.toInstant() |
|
||||||
val type = ArtifactType.valueOf(rows.getString(7).uppercase()) |
|
||||||
val format = ArtifactFormat.valueOf(rows.getString(8).uppercase()) |
|
||||||
val os = OperatingSystem.valueOf(rows.getString(9).uppercase()) |
|
||||||
val arch = Architecture.valueOf(rows.getString(10).uppercase()) |
|
||||||
val jvm = Jvm.valueOf(rows.getString(11).uppercase()) |
|
||||||
val size = rows.getInt(12) |
|
||||||
|
|
||||||
artifacts += ArtifactSummary( |
|
||||||
id, |
|
||||||
game, |
|
||||||
environment, |
|
||||||
build, |
|
||||||
timestamp, |
|
||||||
type, |
|
||||||
format, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
size |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
return@execute artifacts |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun get(id: Long): Artifact? { |
|
||||||
return database.execute { connection -> |
|
||||||
val sources = mutableListOf<ArtifactSource>() |
|
||||||
val links = mutableListOf<ArtifactLinkExport>() |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT DISTINCT name, description, url |
|
||||||
FROM artifact_sources |
|
||||||
WHERE blob_id = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
while (rows.next()) { |
|
||||||
val name = rows.getString(1) |
|
||||||
val description = rows.getString(2) |
|
||||||
val url = rows.getString(3) |
|
||||||
|
|
||||||
sources += ArtifactSource(name, description, url) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT |
|
||||||
a.blob_id, |
|
||||||
a.build_major, |
|
||||||
a.build_minor, |
|
||||||
a.timestamp, |
|
||||||
l.type, |
|
||||||
l.format, |
|
||||||
l.os, |
|
||||||
l.arch, |
|
||||||
l.jvm, |
|
||||||
COALESCE(l.crc32, b.crc32), |
|
||||||
l.sha1, |
|
||||||
COALESCE(l.size, length(b.data)) |
|
||||||
FROM artifact_links l |
|
||||||
LEFT JOIN blobs b ON b.sha1 = l.sha1 |
|
||||||
LEFT JOIN artifacts a ON a.blob_id = b.id |
|
||||||
WHERE l.blob_id = ? |
|
||||||
ORDER BY l.type, l.format, l.os, l.arch, l.jvm |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
while (rows.next()) { |
|
||||||
var linkId: Long? = rows.getLong(1) |
|
||||||
if (rows.wasNull()) { |
|
||||||
linkId = null |
|
||||||
} |
|
||||||
|
|
||||||
var buildMajor: Int? = rows.getInt(2) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMajor = null |
|
||||||
} |
|
||||||
|
|
||||||
var buildMinor: Int? = rows.getInt(3) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMinor = null |
|
||||||
} |
|
||||||
|
|
||||||
val build = if (buildMajor != null) { |
|
||||||
CacheExporter.Build(buildMajor, buildMinor) |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
|
|
||||||
val timestamp = rows.getTimestamp(4)?.toInstant() |
|
||||||
val type = ArtifactType.valueOf(rows.getString(5).uppercase()) |
|
||||||
val format = ArtifactFormat.valueOf(rows.getString(6).uppercase()) |
|
||||||
val os = OperatingSystem.valueOf(rows.getString(7).uppercase()) |
|
||||||
val arch = Architecture.valueOf(rows.getString(8).uppercase()) |
|
||||||
val jvm = Jvm.valueOf(rows.getString(9).uppercase()) |
|
||||||
|
|
||||||
var crc32: Int? = rows.getInt(10) |
|
||||||
if (rows.wasNull()) { |
|
||||||
crc32 = null |
|
||||||
} |
|
||||||
|
|
||||||
val sha1 = rows.getBytes(11) |
|
||||||
|
|
||||||
var size: Int? = rows.getInt(12) |
|
||||||
if (rows.wasNull()) { |
|
||||||
size = null |
|
||||||
} |
|
||||||
|
|
||||||
links += ArtifactLinkExport( |
|
||||||
linkId, |
|
||||||
build, |
|
||||||
timestamp, |
|
||||||
ArtifactLink( |
|
||||||
type, |
|
||||||
format, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
crc32, |
|
||||||
sha1, |
|
||||||
size |
|
||||||
) |
|
||||||
) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT |
|
||||||
g.name, |
|
||||||
e.name, |
|
||||||
a.build_major, |
|
||||||
a.build_minor, |
|
||||||
a.timestamp, |
|
||||||
a.type, |
|
||||||
a.format, |
|
||||||
a.os, |
|
||||||
a.arch, |
|
||||||
a.jvm, |
|
||||||
length(b.data) AS size, |
|
||||||
b.crc32, |
|
||||||
b.sha1 |
|
||||||
FROM artifacts a |
|
||||||
JOIN games g ON g.id = a.game_id |
|
||||||
JOIN environments e ON e.id = a.environment_id |
|
||||||
JOIN blobs b ON b.id = a.blob_id |
|
||||||
WHERE a.blob_id = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
if (!rows.next()) { |
|
||||||
return@execute null |
|
||||||
} |
|
||||||
|
|
||||||
val game = rows.getString(1) |
|
||||||
val environment = rows.getString(2) |
|
||||||
|
|
||||||
var buildMajor: Int? = rows.getInt(3) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMajor = null |
|
||||||
} |
|
||||||
|
|
||||||
var buildMinor: Int? = rows.getInt(4) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMinor = null |
|
||||||
} |
|
||||||
|
|
||||||
val build = if (buildMajor != null) { |
|
||||||
CacheExporter.Build(buildMajor!!, buildMinor) |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
|
|
||||||
val timestamp = rows.getTimestamp(5)?.toInstant() |
|
||||||
val type = ArtifactType.valueOf(rows.getString(6).uppercase()) |
|
||||||
val format = ArtifactFormat.valueOf(rows.getString(7).uppercase()) |
|
||||||
val os = OperatingSystem.valueOf(rows.getString(8).uppercase()) |
|
||||||
val arch = Architecture.valueOf(rows.getString(9).uppercase()) |
|
||||||
val jvm = Jvm.valueOf(rows.getString(10).uppercase()) |
|
||||||
val size = rows.getInt(11) |
|
||||||
val crc32 = rows.getInt(12) |
|
||||||
val sha1 = rows.getBytes(13) |
|
||||||
|
|
||||||
return@execute Artifact( |
|
||||||
ArtifactSummary( |
|
||||||
id, |
|
||||||
game, |
|
||||||
environment, |
|
||||||
build, |
|
||||||
timestamp, |
|
||||||
type, |
|
||||||
format, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
size |
|
||||||
), crc32, sha1, sources, links |
|
||||||
) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun export(id: Long): ArtifactExport? { |
|
||||||
return database.execute { connection -> |
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT |
|
||||||
g.name, |
|
||||||
e.name, |
|
||||||
a.build_major, |
|
||||||
a.build_minor, |
|
||||||
a.timestamp, |
|
||||||
a.type, |
|
||||||
a.format, |
|
||||||
a.os, |
|
||||||
a.arch, |
|
||||||
a.jvm, |
|
||||||
b.data |
|
||||||
FROM artifacts a |
|
||||||
JOIN games g ON g.id = a.game_id |
|
||||||
JOIN environments e ON e.id = a.environment_id |
|
||||||
JOIN blobs b ON b.id = a.blob_id |
|
||||||
WHERE a.blob_id = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
if (!rows.next()) { |
|
||||||
return@execute null |
|
||||||
} |
|
||||||
|
|
||||||
val game = rows.getString(1) |
|
||||||
val environment = rows.getString(2) |
|
||||||
|
|
||||||
var buildMajor: Int? = rows.getInt(3) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMajor = null |
|
||||||
} |
|
||||||
|
|
||||||
var buildMinor: Int? = rows.getInt(4) |
|
||||||
if (rows.wasNull()) { |
|
||||||
buildMinor = null |
|
||||||
} |
|
||||||
|
|
||||||
val build = if (buildMajor != null) { |
|
||||||
CacheExporter.Build(buildMajor, buildMinor) |
|
||||||
} else { |
|
||||||
null |
|
||||||
} |
|
||||||
|
|
||||||
val timestamp = rows.getTimestamp(5)?.toInstant() |
|
||||||
val type = ArtifactType.valueOf(rows.getString(6).uppercase()) |
|
||||||
val format = ArtifactFormat.valueOf(rows.getString(7).uppercase()) |
|
||||||
val os = OperatingSystem.valueOf(rows.getString(8).uppercase()) |
|
||||||
val arch = Architecture.valueOf(rows.getString(9).uppercase()) |
|
||||||
val jvm = Jvm.valueOf(rows.getString(10).uppercase()) |
|
||||||
|
|
||||||
val buf = Unpooled.wrappedBuffer(rows.getBytes(11)) |
|
||||||
val size = buf.readableBytes() |
|
||||||
|
|
||||||
return@execute ArtifactExport( |
|
||||||
ArtifactSummary( |
|
||||||
id, |
|
||||||
game, |
|
||||||
environment, |
|
||||||
build, |
|
||||||
timestamp, |
|
||||||
type, |
|
||||||
format, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
size |
|
||||||
), buf |
|
||||||
) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,997 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import com.github.michaelbull.logging.InlineLogger |
|
||||||
import com.kichik.pecoff4j.PE |
|
||||||
import com.kichik.pecoff4j.constant.MachineType |
|
||||||
import com.kichik.pecoff4j.io.PEParser |
|
||||||
import dorkbox.cabParser.CabParser |
|
||||||
import dorkbox.cabParser.CabStreamSaver |
|
||||||
import dorkbox.cabParser.structure.CabFileEntry |
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import io.netty.buffer.ByteBufAllocator |
|
||||||
import io.netty.buffer.ByteBufInputStream |
|
||||||
import io.netty.buffer.ByteBufOutputStream |
|
||||||
import io.netty.buffer.Unpooled |
|
||||||
import io.netty.util.ByteProcessor |
|
||||||
import jakarta.inject.Inject |
|
||||||
import jakarta.inject.Singleton |
|
||||||
import net.fornwall.jelf.ElfFile |
|
||||||
import net.fornwall.jelf.ElfSymbol |
|
||||||
import org.objectweb.asm.Opcodes |
|
||||||
import org.objectweb.asm.tree.AbstractInsnNode |
|
||||||
import org.objectweb.asm.tree.ClassNode |
|
||||||
import org.objectweb.asm.tree.JumpInsnNode |
|
||||||
import org.objectweb.asm.tree.LdcInsnNode |
|
||||||
import org.objectweb.asm.tree.MethodInsnNode |
|
||||||
import org.objectweb.asm.tree.TypeInsnNode |
|
||||||
import org.openrs2.archive.cache.CacheExporter |
|
||||||
import org.openrs2.archive.cache.CacheImporter |
|
||||||
import org.openrs2.asm.InsnMatcher |
|
||||||
import org.openrs2.asm.classpath.Library |
|
||||||
import org.openrs2.asm.getArgumentExpressions |
|
||||||
import org.openrs2.asm.hasCode |
|
||||||
import org.openrs2.asm.intConstant |
|
||||||
import org.openrs2.asm.io.CabLibraryReader |
|
||||||
import org.openrs2.asm.io.JarLibraryReader |
|
||||||
import org.openrs2.asm.io.LibraryReader |
|
||||||
import org.openrs2.asm.io.Pack200LibraryReader |
|
||||||
import org.openrs2.asm.io.PackClassLibraryReader |
|
||||||
import org.openrs2.asm.nextReal |
|
||||||
import org.openrs2.buffer.use |
|
||||||
import org.openrs2.compress.gzip.Gzip |
|
||||||
import org.openrs2.db.Database |
|
||||||
import org.openrs2.util.io.entries |
|
||||||
import java.io.ByteArrayInputStream |
|
||||||
import java.io.ByteArrayOutputStream |
|
||||||
import java.io.InputStream |
|
||||||
import java.io.OutputStream |
|
||||||
import java.nio.file.Files |
|
||||||
import java.nio.file.Path |
|
||||||
import java.sql.Connection |
|
||||||
import java.sql.Types |
|
||||||
import java.time.Instant |
|
||||||
import java.time.LocalDate |
|
||||||
import java.time.Month |
|
||||||
import java.time.ZoneOffset |
|
||||||
import java.util.jar.JarInputStream |
|
||||||
import java.util.jar.JarOutputStream |
|
||||||
import java.util.jar.Pack200 |
|
||||||
import kotlin.io.path.getLastModifiedTime |
|
||||||
|
|
||||||
@Singleton |
|
||||||
public class ClientImporter @Inject constructor( |
|
||||||
private val database: Database, |
|
||||||
private val alloc: ByteBufAllocator, |
|
||||||
private val packClassLibraryReader: PackClassLibraryReader, |
|
||||||
private val importer: CacheImporter |
|
||||||
) { |
|
||||||
public suspend fun import( |
|
||||||
paths: Iterable<Path>, |
|
||||||
name: String?, |
|
||||||
description: String?, |
|
||||||
url: String?, |
|
||||||
skipErrors: Boolean |
|
||||||
) { |
|
||||||
alloc.buffer().use { buf -> |
|
||||||
for (path in paths) { |
|
||||||
buf.clear() |
|
||||||
|
|
||||||
Files.newInputStream(path).use { input -> |
|
||||||
ByteBufOutputStream(buf).use { output -> |
|
||||||
input.copyTo(output) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
logger.info { "Importing $path" } |
|
||||||
try { |
|
||||||
import( |
|
||||||
parse(buf), |
|
||||||
name, |
|
||||||
description, |
|
||||||
url, |
|
||||||
path.fileName.toString(), |
|
||||||
path.getLastModifiedTime().toInstant() |
|
||||||
) |
|
||||||
} catch (t: Throwable) { |
|
||||||
if (skipErrors) { |
|
||||||
logger.warn(t) { "Failed to import $path" } |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
throw t |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun import( |
|
||||||
artifact: Artifact, |
|
||||||
name: String?, |
|
||||||
description: String?, |
|
||||||
url: String?, |
|
||||||
fileName: String, |
|
||||||
timestamp: Instant |
|
||||||
) { |
|
||||||
database.execute { connection -> |
|
||||||
importer.prepare(connection) |
|
||||||
|
|
||||||
val id = import(connection, artifact) |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
INSERT INTO artifact_sources (blob_id, name, description, url, file_name, timestamp) |
|
||||||
VALUES (?, ?, ?, ?, ?, ?) |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
stmt.setString(2, name) |
|
||||||
stmt.setString(3, description) |
|
||||||
stmt.setString(4, url) |
|
||||||
stmt.setString(5, fileName) |
|
||||||
stmt.setObject(6, timestamp.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE) |
|
||||||
|
|
||||||
stmt.execute() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun import(connection: Connection, artifact: Artifact): Long { |
|
||||||
val id = importer.addBlob(connection, artifact) |
|
||||||
|
|
||||||
val gameId = connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT id |
|
||||||
FROM games |
|
||||||
WHERE name = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setString(1, artifact.game) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
if (!rows.next()) { |
|
||||||
throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
rows.getInt(1) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
val environmentId = connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT id |
|
||||||
FROM environments |
|
||||||
WHERE name = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setString(1, artifact.environment) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
if (!rows.next()) { |
|
||||||
throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
rows.getInt(1) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
INSERT INTO artifacts (blob_id, game_id, environment_id, build_major, build_minor, timestamp, type, format, os, arch, jvm) |
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?::artifact_type, ?::artifact_format, ?::os, ?::arch, ?::jvm) |
|
||||||
ON CONFLICT (blob_id) DO UPDATE SET |
|
||||||
game_id = EXCLUDED.game_id, |
|
||||||
environment_id = EXCLUDED.environment_id, |
|
||||||
build_major = EXCLUDED.build_major, |
|
||||||
build_minor = EXCLUDED.build_minor, |
|
||||||
timestamp = EXCLUDED.timestamp, |
|
||||||
type = EXCLUDED.type, |
|
||||||
format = EXCLUDED.format, |
|
||||||
os = EXCLUDED.os, |
|
||||||
arch = EXCLUDED.arch, |
|
||||||
jvm = EXCLUDED.jvm |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
stmt.setInt(2, gameId) |
|
||||||
stmt.setInt(3, environmentId) |
|
||||||
stmt.setObject(4, artifact.build?.major, Types.INTEGER) |
|
||||||
stmt.setObject(5, artifact.build?.minor, Types.INTEGER) |
|
||||||
stmt.setObject(6, artifact.timestamp?.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE) |
|
||||||
stmt.setString(7, artifact.type.name.lowercase()) |
|
||||||
stmt.setString(8, artifact.format.name.lowercase()) |
|
||||||
stmt.setString(9, artifact.os.name.lowercase()) |
|
||||||
stmt.setString(10, artifact.arch.name.lowercase()) |
|
||||||
stmt.setString(11, artifact.jvm.name.lowercase()) |
|
||||||
|
|
||||||
stmt.execute() |
|
||||||
} |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
DELETE FROM artifact_links |
|
||||||
WHERE blob_id = ? |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setLong(1, id) |
|
||||||
stmt.execute() |
|
||||||
} |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
INSERT INTO artifact_links (blob_id, type, format, os, arch, jvm, sha1, crc32, size) |
|
||||||
VALUES (?, ?::artifact_type, ?::artifact_format, ?::os, ?::arch, ?::jvm, ?, ?, ?) |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
for (link in artifact.links) { |
|
||||||
stmt.setLong(1, id) |
|
||||||
stmt.setString(2, link.type.name.lowercase()) |
|
||||||
stmt.setString(3, link.format.name.lowercase()) |
|
||||||
stmt.setString(4, link.os.name.lowercase()) |
|
||||||
stmt.setString(5, link.arch.name.lowercase()) |
|
||||||
stmt.setString(6, link.jvm.name.lowercase()) |
|
||||||
stmt.setBytes(7, link.sha1) |
|
||||||
stmt.setObject(8, link.crc32, Types.INTEGER) |
|
||||||
stmt.setObject(9, link.size, Types.INTEGER) |
|
||||||
|
|
||||||
stmt.addBatch() |
|
||||||
} |
|
||||||
|
|
||||||
stmt.executeBatch() |
|
||||||
} |
|
||||||
|
|
||||||
return id |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun refresh() { |
|
||||||
data class Blob(val id: Long, val bytes: ByteArray) |
|
||||||
|
|
||||||
database.execute { connection -> |
|
||||||
importer.prepare(connection) |
|
||||||
|
|
||||||
var lastId: Long? = null |
|
||||||
val blobs = mutableListOf<Blob>() |
|
||||||
|
|
||||||
while (true) { |
|
||||||
blobs.clear() |
|
||||||
|
|
||||||
connection.prepareStatement( |
|
||||||
""" |
|
||||||
SELECT a.blob_id, b.data |
|
||||||
FROM artifacts a |
|
||||||
JOIN blobs b ON b.id = a.blob_id |
|
||||||
WHERE ? IS NULL OR a.blob_id > ? |
|
||||||
ORDER BY a.blob_id ASC |
|
||||||
LIMIT 1024 |
|
||||||
""".trimIndent() |
|
||||||
).use { stmt -> |
|
||||||
stmt.setObject(1, lastId, Types.BIGINT) |
|
||||||
stmt.setObject(2, lastId, Types.BIGINT) |
|
||||||
|
|
||||||
stmt.executeQuery().use { rows -> |
|
||||||
while (rows.next()) { |
|
||||||
val id = rows.getLong(1) |
|
||||||
lastId = id |
|
||||||
blobs += Blob(id, rows.getBytes(2)) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
if (blobs.isEmpty()) { |
|
||||||
return@execute |
|
||||||
} |
|
||||||
|
|
||||||
for (blob in blobs) { |
|
||||||
logger.info { "Refreshing artifact ${blob.id}" } |
|
||||||
|
|
||||||
Unpooled.wrappedBuffer(blob.bytes).use { buf -> |
|
||||||
import(connection, parse(buf)) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun parse(buf: ByteBuf): Artifact { |
|
||||||
return if (buf.hasPrefix(JAR)) { |
|
||||||
parseJar(buf) |
|
||||||
} else if (buf.hasPrefix(PACK200)) { |
|
||||||
parsePack200(buf) |
|
||||||
} else if (buf.hasPrefix(CAB)) { |
|
||||||
parseCab(buf) |
|
||||||
} else if ( |
|
||||||
buf.hasPrefix(PACKCLASS_UNCOMPRESSED) || |
|
||||||
buf.hasPrefix(PACKCLASS_BZIP2) || |
|
||||||
buf.hasPrefix(PACKCLASS_GZIP) |
|
||||||
) { |
|
||||||
parseLibrary(buf, packClassLibraryReader, ArtifactFormat.PACKCLASS) |
|
||||||
} else if (buf.hasPrefix(ELF)) { |
|
||||||
parseElf(buf) |
|
||||||
} else if (buf.hasPrefix(PE)) { |
|
||||||
parsePe(buf) |
|
||||||
} else if ( |
|
||||||
buf.hasPrefix(MACHO32BE) || |
|
||||||
buf.hasPrefix(MACHO32LE) || |
|
||||||
buf.hasPrefix(MACHO64BE) || |
|
||||||
buf.hasPrefix(MACHO64LE) || |
|
||||||
buf.hasPrefix(MACHO_UNIVERSAL) |
|
||||||
) { |
|
||||||
parseMachO(buf) |
|
||||||
} else { |
|
||||||
throw IllegalArgumentException() |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseElf(buf: ByteBuf): Artifact { |
|
||||||
val elf = ElfFile.from(ByteBufInputStream(buf.slice())) |
|
||||||
|
|
||||||
val arch = when (elf.e_machine.toInt()) { |
|
||||||
ElfFile.ARCH_i386 -> Architecture.X86 |
|
||||||
ElfFile.ARCH_X86_64 -> Architecture.AMD64 |
|
||||||
ElfFile.ARCH_SPARC -> Architecture.SPARC |
|
||||||
ARCH_SPARCV9 -> Architecture.SPARCV9 |
|
||||||
else -> throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
val comment = String(elf.firstSectionByName(".comment").data) |
|
||||||
val os = if (comment.contains(SOLARIS_COMMENT)) { |
|
||||||
OperatingSystem.SOLARIS |
|
||||||
} else { |
|
||||||
OperatingSystem.LINUX |
|
||||||
} |
|
||||||
|
|
||||||
val symbols = elf.dynamicSymbolTableSection ?: throw IllegalArgumentException() |
|
||||||
val type = getArtifactType(symbols.symbols.asSequence().mapNotNull(ElfSymbol::getName)) |
|
||||||
|
|
||||||
return Artifact( |
|
||||||
buf.retain(), |
|
||||||
"shared", |
|
||||||
"live", |
|
||||||
null, |
|
||||||
null, |
|
||||||
type, |
|
||||||
ArtifactFormat.NATIVE, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
Jvm.SUN, |
|
||||||
emptyList() |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
private fun getArtifactType(symbols: Sequence<String>): ArtifactType { |
|
||||||
for (symbol in symbols) { |
|
||||||
var name = symbol |
|
||||||
if (name.startsWith('_')) { |
|
||||||
name = name.substring(1) |
|
||||||
} |
|
||||||
if (name.startsWith("Java_")) { // RNI methods don't have a Java_ prefix |
|
||||||
name = name.substring("Java_".length) |
|
||||||
} |
|
||||||
|
|
||||||
if (name.startsWith("jaggl_X11_dri_")) { |
|
||||||
return ArtifactType.JAGGL_DRI |
|
||||||
} else if (name.startsWith("jaggl_opengl_")) { |
|
||||||
return ArtifactType.JAGGL |
|
||||||
} else if (name.startsWith("com_sun_opengl_impl_GLImpl_")) { |
|
||||||
return ArtifactType.JOGL |
|
||||||
} else if (name.startsWith("com_sun_opengl_impl_JAWT_")) { |
|
||||||
return ArtifactType.JOGL_AWT |
|
||||||
} else if (name.startsWith("com_sun_gluegen_runtime_")) { |
|
||||||
return ArtifactType.GLUEGEN_RT |
|
||||||
} else if (name.startsWith("jagex3_jagmisc_jagmisc_")) { |
|
||||||
return ArtifactType.JAGMISC |
|
||||||
} else if (name.startsWith("nativeadvert_browsercontrol_")) { |
|
||||||
return ArtifactType.BROWSERCONTROL |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
private fun parsePe(buf: ByteBuf): Artifact { |
|
||||||
val pe = PEParser.parse(ByteBufInputStream(buf.slice())) |
|
||||||
|
|
||||||
val arch = when (pe.coffHeader.machine) { |
|
||||||
MachineType.IMAGE_FILE_MACHINE_I386 -> Architecture.X86 |
|
||||||
MachineType.IMAGE_FILE_MACHINE_AMD64 -> Architecture.AMD64 |
|
||||||
else -> throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
val symbols = parsePeExportNames(buf, pe).toSet() |
|
||||||
|
|
||||||
val type = getArtifactType(symbols.asSequence()) |
|
||||||
val jvm = if (symbols.contains("RNIGetCompatibleVersion")) { |
|
||||||
Jvm.MICROSOFT |
|
||||||
} else { |
|
||||||
Jvm.SUN |
|
||||||
} |
|
||||||
|
|
||||||
return Artifact( |
|
||||||
buf.retain(), |
|
||||||
"shared", |
|
||||||
"live", |
|
||||||
null, |
|
||||||
Instant.ofEpochSecond(pe.coffHeader.timeDateStamp.toLong()), |
|
||||||
type, |
|
||||||
ArtifactFormat.NATIVE, |
|
||||||
OperatingSystem.WINDOWS, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
emptyList() |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parsePeExportNames(buf: ByteBuf, pe: PE): Sequence<String> { |
|
||||||
return sequence { |
|
||||||
val exportTable = pe.imageData.exportTable |
|
||||||
val namePointerTable = |
|
||||||
pe.sectionTable.rvaConverter.convertVirtualAddressToRawDataPointer(exportTable.namePointerRVA.toInt()) |
|
||||||
|
|
||||||
for (i in 0 until exportTable.numberOfNamePointers.toInt()) { |
|
||||||
val namePointerRva = buf.readerIndex() + buf.getIntLE(buf.readerIndex() + namePointerTable + 4 * i) |
|
||||||
val namePointer = pe.sectionTable.rvaConverter.convertVirtualAddressToRawDataPointer(namePointerRva) |
|
||||||
|
|
||||||
val end = buf.forEachByte(namePointer, buf.writerIndex() - namePointer, ByteProcessor.FIND_NUL) |
|
||||||
require(end != -1) { |
|
||||||
"Unterminated string" |
|
||||||
} |
|
||||||
|
|
||||||
yield(buf.toString(namePointer, end - namePointer, Charsets.US_ASCII)) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseMachO(buf: ByteBuf): Artifact { |
|
||||||
val (arch, symbols) = MachO.parse(buf.slice()) |
|
||||||
val type = getArtifactType(symbols.asSequence()) |
|
||||||
|
|
||||||
return Artifact( |
|
||||||
buf.retain(), |
|
||||||
"shared", |
|
||||||
"live", |
|
||||||
null, |
|
||||||
null, |
|
||||||
type, |
|
||||||
ArtifactFormat.NATIVE, |
|
||||||
OperatingSystem.MACOS, |
|
||||||
arch, |
|
||||||
Jvm.SUN, |
|
||||||
emptyList() |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseJar(buf: ByteBuf): Artifact { |
|
||||||
val timestamp = getJarTimestamp(ByteBufInputStream(buf.slice())) |
|
||||||
return parseLibrary(buf, JarLibraryReader, ArtifactFormat.JAR, timestamp) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parsePack200(buf: ByteBuf): Artifact { |
|
||||||
val timestamp = ByteArrayOutputStream().use { tempOutput -> |
|
||||||
Gzip.createHeaderlessInputStream(ByteBufInputStream(buf.slice())).use { gzipInput -> |
|
||||||
JarOutputStream(tempOutput).use { jarOutput -> |
|
||||||
Pack200.newUnpacker().unpack(gzipInput, jarOutput) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
getJarTimestamp(ByteArrayInputStream(tempOutput.toByteArray())) |
|
||||||
} |
|
||||||
|
|
||||||
return parseLibrary(buf, Pack200LibraryReader, ArtifactFormat.PACK200, timestamp) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseCab(buf: ByteBuf): Artifact { |
|
||||||
val timestamp = getCabTimestamp(ByteBufInputStream(buf.slice())) |
|
||||||
return parseLibrary(buf, CabLibraryReader, ArtifactFormat.CAB, timestamp) |
|
||||||
} |
|
||||||
|
|
||||||
private fun getJarTimestamp(input: InputStream): Instant? { |
|
||||||
var timestamp: Instant? = null |
|
||||||
|
|
||||||
JarInputStream(input).use { jar -> |
|
||||||
for (entry in jar.entries) { |
|
||||||
val t = entry.lastModifiedTime?.toInstant() |
|
||||||
if (timestamp == null || (t != null && t < timestamp)) { |
|
||||||
timestamp = t |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return timestamp |
|
||||||
} |
|
||||||
|
|
||||||
private fun getCabTimestamp(input: InputStream): Instant? { |
|
||||||
var timestamp: Instant? = null |
|
||||||
|
|
||||||
CabParser(input, object : CabStreamSaver { |
|
||||||
override fun closeOutputStream(outputStream: OutputStream, entry: CabFileEntry) { |
|
||||||
// entry |
|
||||||
} |
|
||||||
|
|
||||||
override fun openOutputStream(entry: CabFileEntry): OutputStream { |
|
||||||
val t = entry.date.toInstant() |
|
||||||
if (timestamp == null || t < timestamp) { |
|
||||||
timestamp = t |
|
||||||
} |
|
||||||
|
|
||||||
return OutputStream.nullOutputStream() |
|
||||||
} |
|
||||||
|
|
||||||
override fun saveReservedAreaData(data: ByteArray?, dataLength: Int): Boolean { |
|
||||||
return false |
|
||||||
} |
|
||||||
}).extractStream() |
|
||||||
|
|
||||||
return timestamp |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseLibrary( |
|
||||||
buf: ByteBuf, |
|
||||||
reader: LibraryReader, |
|
||||||
format: ArtifactFormat, |
|
||||||
timestamp: Instant? = null |
|
||||||
): Artifact { |
|
||||||
val library = Library.read("client", ByteBufInputStream(buf.slice()), reader) |
|
||||||
|
|
||||||
val game: String |
|
||||||
val build: CacheExporter.Build? |
|
||||||
val type: ArtifactType |
|
||||||
val links: List<ArtifactLink> |
|
||||||
|
|
||||||
val mudclient = library["mudclient"] |
|
||||||
val client = library["client"] |
|
||||||
val loader = library["loader"] |
|
||||||
|
|
||||||
if (mudclient != null) { |
|
||||||
game = "classic" |
|
||||||
build = null // TODO(gpe): classic support |
|
||||||
type = ArtifactType.CLIENT |
|
||||||
links = emptyList() |
|
||||||
} else if (client != null) { |
|
||||||
game = "runescape" |
|
||||||
build = parseClientBuild(library, client) |
|
||||||
type = if (build != null && build.major < COMBINED_BUILD && isClientGl(library)) { |
|
||||||
ArtifactType.CLIENT_GL |
|
||||||
} else { |
|
||||||
ArtifactType.CLIENT |
|
||||||
} |
|
||||||
links = emptyList() |
|
||||||
} else if (loader != null) { |
|
||||||
if (isLoaderClassic(loader)) { |
|
||||||
game = "classic" |
|
||||||
build = null // TODO(gpe): classic support |
|
||||||
type = ArtifactType.LOADER |
|
||||||
links = emptyList() // TODO(gpe): classic support |
|
||||||
} else { |
|
||||||
game = "runescape" |
|
||||||
build = parseSignLinkBuild(library) |
|
||||||
type = if (timestamp != null && timestamp < COMBINED_TIMESTAMP && isLoaderGl(library)) { |
|
||||||
ArtifactType.LOADER_GL |
|
||||||
} else { |
|
||||||
ArtifactType.LOADER |
|
||||||
} |
|
||||||
links = parseLinks(library) |
|
||||||
} |
|
||||||
} else if (library.contains("mapview")) { |
|
||||||
game = "mapview" |
|
||||||
build = null |
|
||||||
type = ArtifactType.CLIENT |
|
||||||
links = emptyList() |
|
||||||
} else if (library.contains("loginapplet")) { |
|
||||||
game = "loginapplet" |
|
||||||
build = null |
|
||||||
type = ArtifactType.CLIENT |
|
||||||
links = emptyList() |
|
||||||
} else if (library.contains("passwordapp")) { |
|
||||||
game = "passapplet" |
|
||||||
build = null |
|
||||||
type = ArtifactType.CLIENT |
|
||||||
links = emptyList() |
|
||||||
} else if (library.contains("jaggl/opengl")) { |
|
||||||
game = "shared" |
|
||||||
type = ArtifactType.JAGGL |
|
||||||
build = null |
|
||||||
links = emptyList() |
|
||||||
} else if (library.contains("com/sun/opengl/impl/GLImpl")) { |
|
||||||
game = "shared" |
|
||||||
type = ArtifactType.JOGL |
|
||||||
build = null |
|
||||||
links = emptyList() |
|
||||||
} else if (library.contains("unpackclass")) { |
|
||||||
game = "shared" |
|
||||||
type = ArtifactType.UNPACKCLASS |
|
||||||
build = null |
|
||||||
links = emptyList() |
|
||||||
} else { |
|
||||||
throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
return Artifact( |
|
||||||
buf.retain(), |
|
||||||
game, |
|
||||||
"live", |
|
||||||
build, |
|
||||||
timestamp, |
|
||||||
type, |
|
||||||
format, |
|
||||||
OperatingSystem.INDEPENDENT, |
|
||||||
Architecture.INDEPENDENT, |
|
||||||
Jvm.INDEPENDENT, |
|
||||||
links |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
private fun isClientGl(library: Library): Boolean { |
|
||||||
for (clazz in library) { |
|
||||||
for (method in clazz.methods) { |
|
||||||
if (!method.hasCode) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (insn in method.instructions) { |
|
||||||
if (insn is MethodInsnNode && insn.name == "glBegin") { |
|
||||||
return true |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
private fun isLoaderClassic(clazz: ClassNode): Boolean { |
|
||||||
for (method in clazz.methods) { |
|
||||||
if (!method.hasCode) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (insn in method.instructions) { |
|
||||||
if (insn is LdcInsnNode && insn.cst == "mudclient") { |
|
||||||
return true |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
private fun isLoaderGl(library: Library): Boolean { |
|
||||||
for (clazz in library) { |
|
||||||
for (method in clazz.methods) { |
|
||||||
if (!method.hasCode || method.name != "<clinit>") { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (insn in method.instructions) { |
|
||||||
if (insn !is LdcInsnNode) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
if (insn.cst == "jaggl.dll" || insn.cst == "jogl.dll") { |
|
||||||
return true |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseClientBuild(library: Library, clazz: ClassNode): CacheExporter.Build? { |
|
||||||
for (method in clazz.methods) { |
|
||||||
if (!method.hasCode || method.name != "main") { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (match in OLD_ENGINE_VERSION_MATCHER.match(method)) { |
|
||||||
val ldc = match[0] as LdcInsnNode |
|
||||||
if (ldc.cst != OLD_ENGINE_VERSION_STRING) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
val version = match[2].intConstant |
|
||||||
if (version != null) { |
|
||||||
return CacheExporter.Build(version, null) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
var betweenNewAndReturn = false |
|
||||||
val candidates = mutableListOf<Int>() |
|
||||||
|
|
||||||
for (insn in method.instructions) { |
|
||||||
if (insn is TypeInsnNode && insn.desc == "client") { |
|
||||||
betweenNewAndReturn = true |
|
||||||
} else if (insn.opcode == Opcodes.RETURN) { |
|
||||||
break |
|
||||||
} else if (betweenNewAndReturn) { |
|
||||||
val candidate = insn.intConstant |
|
||||||
if (candidate != null && candidate in NEW_ENGINE_BUILDS) { |
|
||||||
candidates += candidate |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
for (build in NEW_ENGINE_RESOLUTIONS) { |
|
||||||
candidates -= build |
|
||||||
} |
|
||||||
|
|
||||||
val version = candidates.singleOrNull() |
|
||||||
if (version != null) { |
|
||||||
return CacheExporter.Build(version, null) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return parseSignLinkBuild(library) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseSignLinkBuild(library: Library): CacheExporter.Build? { |
|
||||||
val clazz = library["sign/signlink"] ?: return null |
|
||||||
|
|
||||||
for (field in clazz.fields) { |
|
||||||
val value = field.value |
|
||||||
if (field.name == "clientversion" && field.desc == "I" && value is Int) { |
|
||||||
return CacheExporter.Build(value, null) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return null |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseLinks(library: Library): List<ArtifactLink> { |
|
||||||
val sig = library["sig"] |
|
||||||
if (sig != null) { |
|
||||||
var size: Int? = null |
|
||||||
var sha1: ByteArray? = null |
|
||||||
|
|
||||||
for (field in sig.fields) { |
|
||||||
val value = field.value |
|
||||||
if (field.name == "len" && field.desc == "I" && value is Int) { |
|
||||||
size = value |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
for (method in sig.methods) { |
|
||||||
if (!method.hasCode || method.name != "<clinit>") { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (match in SHA1_MATCHER.match(method)) { |
|
||||||
val len = match[0].intConstant |
|
||||||
if (len != SHA1_BYTES) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
sha1 = ByteArray(SHA1_BYTES) |
|
||||||
for (i in 2 until match.size step 4) { |
|
||||||
val k = match[i + 1].intConstant!! |
|
||||||
val v = match[i + 2].intConstant!! |
|
||||||
sha1[k] = v.toByte() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
require(size != null && sha1 != null) |
|
||||||
|
|
||||||
return listOf( |
|
||||||
ArtifactLink( |
|
||||||
ArtifactType.CLIENT, |
|
||||||
ArtifactFormat.JAR, |
|
||||||
OperatingSystem.INDEPENDENT, |
|
||||||
Architecture.INDEPENDENT, |
|
||||||
Jvm.INDEPENDENT, |
|
||||||
crc32 = null, |
|
||||||
sha1, |
|
||||||
size |
|
||||||
) |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
val loader = library["loader"] |
|
||||||
if (loader != null) { |
|
||||||
val links = mutableListOf<ArtifactLink>() |
|
||||||
val paths = mutableSetOf<String>() |
|
||||||
|
|
||||||
for (method in loader.methods) { |
|
||||||
if (method.name != "run" || method.desc != "()V") { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
for (insn in method.instructions) { |
|
||||||
if (insn !is MethodInsnNode || insn.owner != loader.name || !insn.desc.endsWith(")[B")) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
// TODO(gpe): extract file size too (tricky due to dummy arguments) |
|
||||||
|
|
||||||
val exprs = getArgumentExpressions(insn) ?: continue |
|
||||||
for (expr in exprs) { |
|
||||||
val single = expr.singleOrNull() ?: continue |
|
||||||
if (single !is LdcInsnNode) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
val cst = single.cst |
|
||||||
if (cst is String && FILE_NAME_REGEX.matches(cst)) { |
|
||||||
paths += cst |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
val hashes = mutableMapOf<AbstractInsnNode, ByteArray>() |
|
||||||
|
|
||||||
for (method in loader.methods) { |
|
||||||
for (match in SHA1_CMP_MATCHER.match(method)) { |
|
||||||
val sha1 = ByteArray(SHA1_BYTES) |
|
||||||
var i = 0 |
|
||||||
|
|
||||||
while (i < match.size) { |
|
||||||
var n = match[i++].intConstant |
|
||||||
if (n != null) { |
|
||||||
i++ // ALOAD |
|
||||||
} |
|
||||||
|
|
||||||
val index = match[i++].intConstant!! |
|
||||||
i++ // BALOAD |
|
||||||
|
|
||||||
var xor = false |
|
||||||
if (i + 1 < match.size && match[i + 1].opcode == Opcodes.IXOR) { |
|
||||||
i += 2 // ICONST_M1, IXOR |
|
||||||
xor = true |
|
||||||
} |
|
||||||
|
|
||||||
if (match[i].opcode == Opcodes.IFNE) { |
|
||||||
n = 0 |
|
||||||
i++ |
|
||||||
} else { |
|
||||||
if (n == null) { |
|
||||||
n = match[i++].intConstant!! |
|
||||||
} |
|
||||||
|
|
||||||
i++ // ICMP_IFNE |
|
||||||
} |
|
||||||
|
|
||||||
if (xor) { |
|
||||||
n = n.inv() |
|
||||||
} |
|
||||||
|
|
||||||
sha1[index] = n.toByte() |
|
||||||
} |
|
||||||
|
|
||||||
hashes[match[0]] = sha1 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
for (method in loader.methods) { |
|
||||||
for (match in PATH_CMP_MATCHER.match(method)) { |
|
||||||
val first = match[0] |
|
||||||
val ldc = if (first is LdcInsnNode) { |
|
||||||
first |
|
||||||
} else { |
|
||||||
match[1] as LdcInsnNode |
|
||||||
} |
|
||||||
|
|
||||||
val path = ldc.cst |
|
||||||
if (path !is String) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
val acmp = match[2] as JumpInsnNode |
|
||||||
val target = if (acmp.opcode == Opcodes.IF_ACMPNE) { |
|
||||||
acmp.nextReal |
|
||||||
} else { |
|
||||||
acmp.label.nextReal |
|
||||||
} |
|
||||||
|
|
||||||
val hash = hashes.remove(target) ?: continue |
|
||||||
if (!paths.remove(path)) { |
|
||||||
logger.warn { "Adding link for unused file $path" } |
|
||||||
} |
|
||||||
|
|
||||||
links += parseLink(path, hash) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
if (paths.size != hashes.size || paths.size > 1) { |
|
||||||
throw IllegalArgumentException() |
|
||||||
} else if (paths.size == 1) { |
|
||||||
links += parseLink(paths.single(), hashes.values.single()) |
|
||||||
} |
|
||||||
|
|
||||||
return links |
|
||||||
} |
|
||||||
|
|
||||||
// TODO(gpe) |
|
||||||
return emptyList() |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseLink(path: String, sha1: ByteArray): ArtifactLink { |
|
||||||
val m = FILE_NAME_REGEX.matchEntire(path) ?: throw IllegalArgumentException() |
|
||||||
val (name, crc1, ext, crc2) = m.destructured |
|
||||||
|
|
||||||
val type = when (name) { |
|
||||||
// TODO(gpe): funorb loaders |
|
||||||
"runescape", "client" -> ArtifactType.CLIENT |
|
||||||
"unpackclass" -> ArtifactType.UNPACKCLASS |
|
||||||
"jogl", "jogltrimmed" -> ArtifactType.JOGL |
|
||||||
"jogl_awt" -> ArtifactType.JOGL_AWT |
|
||||||
else -> throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
val format = when (ext) { |
|
||||||
"pack200" -> ArtifactFormat.PACK200 |
|
||||||
"js5" -> ArtifactFormat.PACKCLASS |
|
||||||
"jar", "pack" -> ArtifactFormat.JAR |
|
||||||
"dll" -> ArtifactFormat.NATIVE |
|
||||||
else -> throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
val os = if (format == ArtifactFormat.NATIVE) OperatingSystem.WINDOWS else OperatingSystem.INDEPENDENT |
|
||||||
val arch = if (format == ArtifactFormat.NATIVE) Architecture.X86 else Architecture.INDEPENDENT |
|
||||||
val jvm = if (format == ArtifactFormat.NATIVE) Jvm.SUN else Jvm.INDEPENDENT |
|
||||||
|
|
||||||
val crc = crc1.toIntOrNull() ?: crc2.toIntOrNull() ?: throw IllegalArgumentException() |
|
||||||
|
|
||||||
return ArtifactLink( |
|
||||||
type, |
|
||||||
format, |
|
||||||
os, |
|
||||||
arch, |
|
||||||
jvm, |
|
||||||
crc, |
|
||||||
sha1, |
|
||||||
null |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
private fun ByteBuf.hasPrefix(bytes: ByteArray): Boolean { |
|
||||||
Unpooled.wrappedBuffer(bytes).use { prefix -> |
|
||||||
val len = prefix.readableBytes() |
|
||||||
if (readableBytes() < len) { |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
return slice(readerIndex(), len) == prefix |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private val logger = InlineLogger() |
|
||||||
|
|
||||||
private val CAB = byteArrayOf('M'.code.toByte(), 'S'.code.toByte(), 'C'.code.toByte(), 'F'.code.toByte()) |
|
||||||
private val ELF = byteArrayOf(0x7F, 'E'.code.toByte(), 'L'.code.toByte(), 'F'.code.toByte()) |
|
||||||
private val JAR = byteArrayOf('P'.code.toByte(), 'K'.code.toByte(), 0x03, 0x04) |
|
||||||
private val MACHO32BE = byteArrayOf(0xFE.toByte(), 0xED.toByte(), 0xFA.toByte(), 0xCE.toByte()) |
|
||||||
private val MACHO32LE = byteArrayOf(0xCE.toByte(), 0xFA.toByte(), 0xED.toByte(), 0xFE.toByte()) |
|
||||||
private val MACHO64BE = byteArrayOf(0xFE.toByte(), 0xED.toByte(), 0xFA.toByte(), 0xCF.toByte()) |
|
||||||
private val MACHO64LE = byteArrayOf(0xCF.toByte(), 0xFA.toByte(), 0xED.toByte(), 0xFE.toByte()) |
|
||||||
private val MACHO_UNIVERSAL = byteArrayOf(0xCA.toByte(), 0xFE.toByte(), 0xBA.toByte(), 0xBE.toByte()) |
|
||||||
private val PACK200 = byteArrayOf(0x08) |
|
||||||
private val PACKCLASS_UNCOMPRESSED = byteArrayOf(0x00) |
|
||||||
private val PACKCLASS_BZIP2 = byteArrayOf(0x01) |
|
||||||
private val PACKCLASS_GZIP = byteArrayOf(0x02) |
|
||||||
private val PE = byteArrayOf('M'.code.toByte(), 'Z'.code.toByte()) |
|
||||||
|
|
||||||
private const val OLD_ENGINE_VERSION_STRING = "RS2 user client - release #" |
|
||||||
private val OLD_ENGINE_VERSION_MATCHER = |
|
||||||
InsnMatcher.compile("LDC INVOKESPECIAL (ICONST | BIPUSH | SIPUSH | LDC)") |
|
||||||
|
|
||||||
private val NEW_ENGINE_RESOLUTIONS = listOf(765, 503, 1024, 768) |
|
||||||
private val NEW_ENGINE_BUILDS = 402..916 |
|
||||||
|
|
||||||
private const val COMBINED_BUILD = 555 |
|
||||||
private val COMBINED_TIMESTAMP = LocalDate.of(2009, Month.SEPTEMBER, 2) |
|
||||||
.atStartOfDay(ZoneOffset.UTC) |
|
||||||
.toInstant() |
|
||||||
|
|
||||||
private const val ARCH_SPARCV9 = 43 |
|
||||||
private const val SOLARIS_COMMENT = "Solaris Link Editors:" |
|
||||||
|
|
||||||
private const val SHA1_BYTES = 20 |
|
||||||
private val SHA1_MATCHER = |
|
||||||
InsnMatcher.compile("BIPUSH NEWARRAY (DUP (ICONST | BIPUSH) (ICONST | BIPUSH | SIPUSH) IASTORE)+") |
|
||||||
|
|
||||||
private val FILE_NAME_REGEX = Regex("([a-z_]+)(?:_(-?[0-9]+))?[.]([a-z0-9]+)(?:\\?crc=(-?[0-9]+))?") |
|
||||||
private val SHA1_CMP_MATCHER = |
|
||||||
InsnMatcher.compile("((ICONST | BIPUSH)? ALOAD (ICONST | BIPUSH) BALOAD (ICONST IXOR)? (ICONST | BIPUSH)? (IF_ICMPEQ | IF_ICMPNE | IFEQ | IFNE))+") |
|
||||||
private val PATH_CMP_MATCHER = InsnMatcher.compile("(LDC ALOAD | ALOAD LDC) (IF_ACMPEQ | IF_ACMPNE)") |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,30 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.CliktCommand |
|
||||||
import com.github.ajalt.clikt.parameters.arguments.argument |
|
||||||
import com.github.ajalt.clikt.parameters.types.defaultStdout |
|
||||||
import com.github.ajalt.clikt.parameters.types.long |
|
||||||
import com.github.ajalt.clikt.parameters.types.outputStream |
|
||||||
import com.google.inject.Guice |
|
||||||
import kotlinx.coroutines.runBlocking |
|
||||||
import org.openrs2.archive.ArchiveModule |
|
||||||
import org.openrs2.inject.CloseableInjector |
|
||||||
import java.io.FileNotFoundException |
|
||||||
|
|
||||||
public class ExportCommand : CliktCommand(name = "export") { |
|
||||||
private val id by argument().long() |
|
||||||
private val output by argument().outputStream().defaultStdout() |
|
||||||
|
|
||||||
override fun run(): Unit = runBlocking { |
|
||||||
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> |
|
||||||
val exporter = injector.getInstance(ClientExporter::class.java) |
|
||||||
val artifact = exporter.export(id) ?: throw FileNotFoundException() |
|
||||||
try { |
|
||||||
val buf = artifact.content() |
|
||||||
buf.readBytes(output, buf.readableBytes()) |
|
||||||
} finally { |
|
||||||
artifact.release() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,32 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.CliktCommand |
|
||||||
import com.github.ajalt.clikt.parameters.arguments.argument |
|
||||||
import com.github.ajalt.clikt.parameters.arguments.multiple |
|
||||||
import com.github.ajalt.clikt.parameters.options.flag |
|
||||||
import com.github.ajalt.clikt.parameters.options.option |
|
||||||
import com.github.ajalt.clikt.parameters.types.path |
|
||||||
import com.google.inject.Guice |
|
||||||
import kotlinx.coroutines.runBlocking |
|
||||||
import org.openrs2.archive.ArchiveModule |
|
||||||
import org.openrs2.inject.CloseableInjector |
|
||||||
|
|
||||||
public class ImportCommand : CliktCommand(name = "import") { |
|
||||||
private val name by option() |
|
||||||
private val description by option() |
|
||||||
private val url by option() |
|
||||||
private val skipErrors by option().flag() |
|
||||||
|
|
||||||
private val input by argument().path( |
|
||||||
mustExist = true, |
|
||||||
canBeDir = false, |
|
||||||
mustBeReadable = true, |
|
||||||
).multiple() |
|
||||||
|
|
||||||
override fun run(): Unit = runBlocking { |
|
||||||
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> |
|
||||||
val importer = injector.getInstance(ClientImporter::class.java) |
|
||||||
importer.import(input, name, description, url, skipErrors) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,7 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
public enum class Jvm { |
|
||||||
INDEPENDENT, |
|
||||||
SUN, |
|
||||||
MICROSOFT |
|
||||||
} |
|
||||||
@ -1,116 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import org.openrs2.buffer.readString |
|
||||||
|
|
||||||
public data class MachO( |
|
||||||
public val architecture: Architecture, |
|
||||||
public val symbols: Set<String>, |
|
||||||
) { |
|
||||||
public companion object { |
|
||||||
private const val MACHO_UNIVERSAL = 0xCAFEBABE.toInt() |
|
||||||
private const val MACHO32BE = 0xFEEDFACE.toInt() |
|
||||||
private const val MACHO32LE = 0xCEFAEDFE.toInt() |
|
||||||
private const val MACHO64BE = 0xFEEDFACF.toInt() |
|
||||||
private const val MACHO64LE = 0xCFFAEDFE.toInt() |
|
||||||
|
|
||||||
private const val CPU_TYPE_X86 = 0x7 |
|
||||||
private const val CPU_TYPE_AMD64 = 0x1000007 |
|
||||||
private const val CPU_TYPE_POWERPC = 0x12 |
|
||||||
|
|
||||||
private const val COMMAND_SYMTAB = 0x2 |
|
||||||
|
|
||||||
public fun parse(buf: ByteBuf): MachO { |
|
||||||
val magic = buf.getInt(buf.readerIndex()) |
|
||||||
return if (magic == MACHO_UNIVERSAL) { |
|
||||||
parseFat(buf) |
|
||||||
} else { |
|
||||||
parseMachO(buf) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseFat(buf: ByteBuf): MachO { |
|
||||||
buf.skipBytes(4) |
|
||||||
|
|
||||||
val symbols = mutableSetOf<String>() |
|
||||||
val count = buf.readInt() |
|
||||||
|
|
||||||
for (i in 0 until count) { |
|
||||||
buf.skipBytes(8) |
|
||||||
|
|
||||||
val offset = buf.readInt() |
|
||||||
val size = buf.readInt() |
|
||||||
|
|
||||||
buf.skipBytes(4) |
|
||||||
|
|
||||||
symbols += parseMachO(buf.slice(offset, size)).symbols |
|
||||||
} |
|
||||||
|
|
||||||
return MachO(Architecture.UNIVERSAL, symbols) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseMachO(buf: ByteBuf): MachO { |
|
||||||
val magic = buf.readInt() |
|
||||||
require(magic == MACHO32BE || magic == MACHO32LE || magic == MACHO64BE || magic == MACHO64LE) |
|
||||||
|
|
||||||
val big = magic == MACHO32BE || magic == MACHO64BE |
|
||||||
val x64 = magic == MACHO64LE || magic == MACHO64BE |
|
||||||
|
|
||||||
val arch = when (if (big) buf.readInt() else buf.readIntLE()) { |
|
||||||
CPU_TYPE_X86 -> Architecture.X86 |
|
||||||
CPU_TYPE_AMD64 -> Architecture.AMD64 |
|
||||||
CPU_TYPE_POWERPC -> Architecture.POWERPC |
|
||||||
else -> throw IllegalArgumentException() |
|
||||||
} |
|
||||||
|
|
||||||
buf.skipBytes(4) // cpuSubType |
|
||||||
buf.skipBytes(4) // fileType |
|
||||||
|
|
||||||
val nCmds = if (big) buf.readInt() else buf.readIntLE() |
|
||||||
|
|
||||||
buf.skipBytes(4) // sizeOfCmds |
|
||||||
buf.skipBytes(4) // flags |
|
||||||
|
|
||||||
if (x64) { |
|
||||||
buf.skipBytes(4) // reserved |
|
||||||
} |
|
||||||
|
|
||||||
val symbols = parseCommands(buf, big, nCmds) |
|
||||||
|
|
||||||
return MachO(arch, symbols) |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseCommands(buf: ByteBuf, big: Boolean, count: Int): Set<String> { |
|
||||||
for (i in 0 until count) { |
|
||||||
val base = buf.readerIndex() |
|
||||||
|
|
||||||
val command = if (big) buf.readInt() else buf.readIntLE() |
|
||||||
val size = if (big) buf.readInt() else buf.readIntLE() |
|
||||||
|
|
||||||
if (command == COMMAND_SYMTAB) { |
|
||||||
buf.skipBytes(8) |
|
||||||
|
|
||||||
val strOff = if (big) buf.readInt() else buf.readIntLE() |
|
||||||
val strSize = if (big) buf.readInt() else buf.readIntLE() |
|
||||||
|
|
||||||
return parseStringTable(buf.slice(strOff, strSize)) |
|
||||||
} |
|
||||||
|
|
||||||
buf.readerIndex(base + size) |
|
||||||
} |
|
||||||
|
|
||||||
return emptySet() |
|
||||||
} |
|
||||||
|
|
||||||
private fun parseStringTable(buf: ByteBuf): Set<String> { |
|
||||||
return buildSet { |
|
||||||
while (buf.isReadable) { |
|
||||||
val str = buf.readString(Charsets.US_ASCII) |
|
||||||
if (str.isNotEmpty()) { |
|
||||||
add(str) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,43 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import io.ktor.http.ContentType |
|
||||||
|
|
||||||
public enum class OperatingSystem { |
|
||||||
INDEPENDENT, |
|
||||||
WINDOWS, |
|
||||||
MACOS, |
|
||||||
LINUX, |
|
||||||
SOLARIS; |
|
||||||
|
|
||||||
public fun getPrefix(): String { |
|
||||||
return when (this) { |
|
||||||
INDEPENDENT -> throw IllegalArgumentException() |
|
||||||
WINDOWS -> "" |
|
||||||
else -> "lib" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public fun getExtension(): String { |
|
||||||
return when (this) { |
|
||||||
INDEPENDENT -> throw IllegalArgumentException() |
|
||||||
WINDOWS -> "dll" |
|
||||||
MACOS -> "dylib" |
|
||||||
LINUX, SOLARIS -> "so" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
public fun getContentType(): ContentType { |
|
||||||
return when (this) { |
|
||||||
INDEPENDENT -> throw IllegalArgumentException() |
|
||||||
WINDOWS -> PE |
|
||||||
MACOS -> MACHO |
|
||||||
LINUX, SOLARIS -> ELF_SHARED |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private val ELF_SHARED = ContentType("application", "x-sharedlib") |
|
||||||
private val MACHO = ContentType("application", "x-mach-binary") |
|
||||||
private val PE = ContentType("application", "vnd.microsoft.portable-executable") |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,16 +0,0 @@ |
|||||||
package org.openrs2.archive.client |
|
||||||
|
|
||||||
import com.github.ajalt.clikt.core.CliktCommand |
|
||||||
import com.google.inject.Guice |
|
||||||
import kotlinx.coroutines.runBlocking |
|
||||||
import org.openrs2.archive.ArchiveModule |
|
||||||
import org.openrs2.inject.CloseableInjector |
|
||||||
|
|
||||||
public class RefreshCommand : CliktCommand(name = "refresh") { |
|
||||||
override fun run(): Unit = runBlocking { |
|
||||||
CloseableInjector(Guice.createInjector(ArchiveModule)).use { injector -> |
|
||||||
val importer = injector.getInstance(ClientImporter::class.java) |
|
||||||
importer.refresh() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,57 +0,0 @@ |
|||||||
package org.openrs2.archive.key |
|
||||||
|
|
||||||
import jakarta.inject.Inject |
|
||||||
import jakarta.inject.Singleton |
|
||||||
import kotlinx.coroutines.Dispatchers |
|
||||||
import kotlinx.coroutines.future.await |
|
||||||
import kotlinx.coroutines.withContext |
|
||||||
import org.openrs2.crypto.SymmetricKey |
|
||||||
import org.openrs2.http.checkStatusCode |
|
||||||
import java.net.URI |
|
||||||
import java.net.http.HttpClient |
|
||||||
import java.net.http.HttpRequest |
|
||||||
import java.net.http.HttpResponse |
|
||||||
import java.time.Duration |
|
||||||
|
|
||||||
@Singleton |
|
||||||
public class HdosKeyDownloader @Inject constructor( |
|
||||||
private val client: HttpClient |
|
||||||
) : KeyDownloader(KeySource.HDOS) { |
|
||||||
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> { |
|
||||||
return setOf(ENDPOINT) |
|
||||||
} |
|
||||||
|
|
||||||
override suspend fun download(url: String): Sequence<SymmetricKey> { |
|
||||||
val request = HttpRequest.newBuilder(URI(url)) |
|
||||||
.GET() |
|
||||||
.timeout(Duration.ofSeconds(30)) |
|
||||||
.build() |
|
||||||
|
|
||||||
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() |
|
||||||
response.checkStatusCode() |
|
||||||
|
|
||||||
return withContext(Dispatchers.IO) { |
|
||||||
response.body().use { input -> |
|
||||||
input.bufferedReader().use { reader -> |
|
||||||
val keys = mutableSetOf<SymmetricKey>() |
|
||||||
|
|
||||||
for (line in reader.lineSequence()) { |
|
||||||
val parts = line.split(',') |
|
||||||
if (parts.size < 3) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
val key = SymmetricKey.fromHexOrNull(parts[2]) ?: continue |
|
||||||
keys += key |
|
||||||
} |
|
||||||
|
|
||||||
keys.asSequence() |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
private companion object { |
|
||||||
private const val ENDPOINT = "https://api.hdos.dev/keys/get" |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,13 +1,13 @@ |
|||||||
package org.openrs2.archive.key |
package org.openrs2.archive.key |
||||||
|
|
||||||
import org.openrs2.crypto.SymmetricKey |
import org.openrs2.crypto.XteaKey |
||||||
import java.io.InputStream |
import java.io.InputStream |
||||||
|
|
||||||
public object HexKeyReader : KeyReader { |
public object HexKeyReader : KeyReader { |
||||||
override fun read(input: InputStream): Sequence<SymmetricKey> { |
override fun read(input: InputStream): Sequence<XteaKey> { |
||||||
return input.bufferedReader() |
return input.bufferedReader() |
||||||
.lineSequence() |
.lineSequence() |
||||||
.map(SymmetricKey::fromHexOrNull) |
.map(XteaKey::fromHexOrNull) |
||||||
.filterNotNull() |
.filterNotNull() |
||||||
} |
} |
||||||
} |
} |
||||||
|
|||||||
@ -1,10 +1,10 @@ |
|||||||
package org.openrs2.archive.key |
package org.openrs2.archive.key |
||||||
|
|
||||||
import org.openrs2.crypto.SymmetricKey |
import org.openrs2.crypto.XteaKey |
||||||
|
|
||||||
public abstract class KeyDownloader( |
public abstract class KeyDownloader( |
||||||
public val source: KeySource |
public val source: KeySource |
||||||
) { |
) { |
||||||
public abstract suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> |
public abstract suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> |
||||||
public abstract suspend fun download(url: String): Sequence<SymmetricKey> |
public abstract suspend fun download(url: String): Sequence<XteaKey> |
||||||
} |
} |
||||||
|
|||||||
@ -1,8 +1,8 @@ |
|||||||
package org.openrs2.archive.key |
package org.openrs2.archive.key |
||||||
|
|
||||||
import org.openrs2.crypto.SymmetricKey |
import org.openrs2.crypto.XteaKey |
||||||
import java.io.InputStream |
import java.io.InputStream |
||||||
|
|
||||||
public interface KeyReader { |
public interface KeyReader { |
||||||
public fun read(input: InputStream): Sequence<SymmetricKey> |
public fun read(input: InputStream): Sequence<XteaKey> |
||||||
} |
} |
||||||
|
|||||||
@ -0,0 +1,19 @@ |
|||||||
|
package org.openrs2.archive.key |
||||||
|
|
||||||
|
import java.net.http.HttpClient |
||||||
|
import javax.inject.Inject |
||||||
|
import javax.inject.Singleton |
||||||
|
|
||||||
|
@Singleton |
||||||
|
public class OpenOsrsKeyDownloader @Inject constructor( |
||||||
|
client: HttpClient, |
||||||
|
jsonKeyReader: JsonKeyReader |
||||||
|
) : JsonKeyDownloader(KeySource.OPENOSRS, client, jsonKeyReader) { |
||||||
|
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> { |
||||||
|
return setOf(ENDPOINT) |
||||||
|
} |
||||||
|
|
||||||
|
private companion object { |
||||||
|
private const val ENDPOINT = "https://xtea.openosrs.dev/get" |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,50 @@ |
|||||||
|
package org.openrs2.archive.key |
||||||
|
|
||||||
|
import kotlinx.coroutines.Dispatchers |
||||||
|
import kotlinx.coroutines.future.await |
||||||
|
import kotlinx.coroutines.withContext |
||||||
|
import org.jsoup.Jsoup |
||||||
|
import org.openrs2.http.charset |
||||||
|
import org.openrs2.http.checkStatusCode |
||||||
|
import java.net.URI |
||||||
|
import java.net.http.HttpClient |
||||||
|
import java.net.http.HttpRequest |
||||||
|
import java.net.http.HttpResponse |
||||||
|
import java.time.Duration |
||||||
|
import javax.inject.Inject |
||||||
|
import javax.inject.Singleton |
||||||
|
|
||||||
|
@Singleton |
||||||
|
public class PolarKeyDownloader @Inject constructor( |
||||||
|
private val client: HttpClient, |
||||||
|
jsonKeyReader: JsonKeyReader |
||||||
|
) : JsonKeyDownloader(KeySource.POLAR, client, jsonKeyReader) { |
||||||
|
override suspend fun getMissingUrls(seenUrls: Set<String>): Set<String> { |
||||||
|
val request = HttpRequest.newBuilder(ENDPOINT) |
||||||
|
.GET() |
||||||
|
.timeout(Duration.ofSeconds(30)) |
||||||
|
.build() |
||||||
|
|
||||||
|
val response = client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream()).await() |
||||||
|
response.checkStatusCode() |
||||||
|
|
||||||
|
val document = withContext(Dispatchers.IO) { |
||||||
|
Jsoup.parse(response.body(), response.charset?.name(), ENDPOINT.toString()) |
||||||
|
} |
||||||
|
|
||||||
|
val urls = mutableSetOf<String>() |
||||||
|
|
||||||
|
for (element in document.select("a")) { |
||||||
|
val url = element.absUrl("href") |
||||||
|
if (url.endsWith(".json") && url !in seenUrls) { |
||||||
|
urls += url |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return urls |
||||||
|
} |
||||||
|
|
||||||
|
private companion object { |
||||||
|
private val ENDPOINT = URI("https://archive.runestats.com/osrs/xtea/") |
||||||
|
} |
||||||
|
} |
||||||
@ -1,82 +0,0 @@ |
|||||||
package org.openrs2.archive.web |
|
||||||
|
|
||||||
import io.ktor.http.ContentDisposition |
|
||||||
import io.ktor.http.HttpHeaders |
|
||||||
import io.ktor.http.HttpStatusCode |
|
||||||
import io.ktor.server.application.ApplicationCall |
|
||||||
import io.ktor.server.response.header |
|
||||||
import io.ktor.server.response.respond |
|
||||||
import io.ktor.server.response.respondOutputStream |
|
||||||
import io.ktor.server.thymeleaf.ThymeleafContent |
|
||||||
import jakarta.inject.Inject |
|
||||||
import jakarta.inject.Singleton |
|
||||||
import org.openrs2.archive.client.ClientExporter |
|
||||||
|
|
||||||
@Singleton |
|
||||||
public class ClientsController @Inject constructor( |
|
||||||
private val exporter: ClientExporter |
|
||||||
) { |
|
||||||
public suspend fun index(call: ApplicationCall) { |
|
||||||
val artifacts = exporter.list() |
|
||||||
|
|
||||||
call.respond( |
|
||||||
ThymeleafContent( |
|
||||||
"clients/index.html", mapOf( |
|
||||||
"artifacts" to artifacts |
|
||||||
) |
|
||||||
) |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun show(call: ApplicationCall) { |
|
||||||
val id = call.parameters["id"]?.toLongOrNull() |
|
||||||
if (id == null) { |
|
||||||
call.respond(HttpStatusCode.NotFound) |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
val artifact = exporter.get(id) |
|
||||||
if (artifact == null) { |
|
||||||
call.respond(HttpStatusCode.NotFound) |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
call.respond( |
|
||||||
ThymeleafContent( |
|
||||||
"clients/show.html", mapOf( |
|
||||||
"artifact" to artifact |
|
||||||
) |
|
||||||
) |
|
||||||
) |
|
||||||
} |
|
||||||
|
|
||||||
public suspend fun export(call: ApplicationCall) { |
|
||||||
val id = call.parameters["id"]?.toLongOrNull() |
|
||||||
if (id == null) { |
|
||||||
call.respond(HttpStatusCode.NotFound) |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
val artifact = exporter.export(id) |
|
||||||
if (artifact == null) { |
|
||||||
call.respond(HttpStatusCode.NotFound) |
|
||||||
return |
|
||||||
} |
|
||||||
|
|
||||||
call.response.header( |
|
||||||
HttpHeaders.ContentLength, |
|
||||||
artifact.summary.size.toString() |
|
||||||
) |
|
||||||
|
|
||||||
call.response.header( |
|
||||||
HttpHeaders.ContentDisposition, |
|
||||||
ContentDisposition.Attachment |
|
||||||
.withParameter(ContentDisposition.Parameters.FileName, artifact.summary.name) |
|
||||||
.toString() |
|
||||||
) |
|
||||||
|
|
||||||
call.respondOutputStream(artifact.summary.format.getContentType(artifact.summary.os)) { |
|
||||||
artifact.content().readBytes(this, artifact.summary.size) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,31 +0,0 @@ |
|||||||
package org.openrs2.archive.world |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
import org.openrs2.buffer.readString |
|
||||||
|
|
||||||
public data class World( |
|
||||||
public val id: Int, |
|
||||||
public val flags: Int, |
|
||||||
public val hostname: String, |
|
||||||
public val activity: String, |
|
||||||
public val country: Int, |
|
||||||
public val players: Int |
|
||||||
) { |
|
||||||
public val isBeta: Boolean |
|
||||||
get() = (flags and FLAG_BETA) != 0 |
|
||||||
|
|
||||||
public companion object { |
|
||||||
private const val FLAG_BETA = 0x10000 |
|
||||||
|
|
||||||
public fun read(buf: ByteBuf): World { |
|
||||||
val id = buf.readUnsignedShort() |
|
||||||
val flags = buf.readInt() |
|
||||||
val hostname = buf.readString() |
|
||||||
val activity = buf.readString() |
|
||||||
val country = buf.readUnsignedByte().toInt() |
|
||||||
val players = buf.readShort().toInt() |
|
||||||
|
|
||||||
return World(id, flags, hostname, activity, country, players) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,22 +0,0 @@ |
|||||||
package org.openrs2.archive.world |
|
||||||
|
|
||||||
import io.netty.buffer.ByteBuf |
|
||||||
|
|
||||||
public data class WorldList( |
|
||||||
public val worlds: List<World> |
|
||||||
) { |
|
||||||
public companion object { |
|
||||||
public fun read(buf: ByteBuf): WorldList { |
|
||||||
buf.skipBytes(4) |
|
||||||
|
|
||||||
val count = buf.readUnsignedShort() |
|
||||||
val worlds = buildList(count) { |
|
||||||
for (i in 0 until count) { |
|
||||||
add(World.read(buf)) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return WorldList(worlds) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
@ -1,176 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
CREATE TABLE scopes ( |
|
||||||
id SERIAL PRIMARY KEY NOT NULL, |
|
||||||
name TEXT UNIQUE NOT NULL |
|
||||||
); |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('runescape'); |
|
||||||
|
|
||||||
ALTER TABLE games |
|
||||||
ADD COLUMN scope_id INTEGER DEFAULT 1 NOT NULL REFERENCES scopes (id); |
|
||||||
|
|
||||||
ALTER TABLE games |
|
||||||
ALTER COLUMN scope_id DROP DEFAULT; |
|
||||||
|
|
||||||
-- XXX(gpe): I don't think we can easily replace this as the source_groups |
|
||||||
-- table doesn't contain a scope_id directly - only indirectly via the sources |
|
||||||
-- and games tables. |
|
||||||
ALTER TABLE source_groups |
|
||||||
DROP CONSTRAINT source_groups_archive_id_group_id_version_version_truncate_fkey; |
|
||||||
|
|
||||||
ALTER TABLE groups |
|
||||||
ADD COLUMN scope_id INTEGER DEFAULT 1 NOT NULL REFERENCES scopes (id), |
|
||||||
DROP CONSTRAINT groups_pkey, |
|
||||||
ADD PRIMARY KEY (scope_id, archive_id, group_id, version, version_truncated, container_id); |
|
||||||
|
|
||||||
ALTER TABLE groups |
|
||||||
ALTER COLUMN scope_id DROP DEFAULT; |
|
||||||
|
|
||||||
CREATE FUNCTION resolve_index(_scope_id INTEGER, _archive_id uint1, _crc32 INTEGER, _version INTEGER) RETURNS SETOF containers AS $$ |
|
||||||
SELECT c.* |
|
||||||
FROM groups g |
|
||||||
JOIN containers c ON c.id = g.container_id |
|
||||||
JOIN indexes i ON i.container_id = c.id |
|
||||||
WHERE g.scope_id = _scope_id AND g.archive_id = 255 AND g.group_id = _archive_id::INTEGER AND c.crc32 = _crc32 AND |
|
||||||
g.version = _version AND NOT g.version_truncated AND i.version = _version |
|
||||||
ORDER BY c.id ASC |
|
||||||
LIMIT 1; |
|
||||||
$$ LANGUAGE SQL STABLE PARALLEL SAFE ROWS 1; |
|
||||||
|
|
||||||
CREATE FUNCTION resolve_group(_scope_id INTEGER, _archive_id uint1, _group_id INTEGER, _crc32 INTEGER, _version INTEGER) RETURNS SETOF containers AS $$ |
|
||||||
SELECT c.* |
|
||||||
FROM groups g |
|
||||||
JOIN containers c ON c.id = g.container_id |
|
||||||
WHERE g.scope_id = _scope_id AND g.archive_id = _archive_id AND g.group_id = _group_id AND c.crc32 = _crc32 AND ( |
|
||||||
(g.version = _version AND NOT g.version_truncated) OR |
|
||||||
(g.version = _version & 65535 AND g.version_truncated) |
|
||||||
) |
|
||||||
ORDER BY g.version_truncated ASC, c.id ASC |
|
||||||
LIMIT 1; |
|
||||||
$$ LANGUAGE SQL STABLE PARALLEL SAFE ROWS 1; |
|
||||||
|
|
||||||
DROP VIEW resolved_groups; |
|
||||||
DROP VIEW resolved_indexes; |
|
||||||
|
|
||||||
CREATE VIEW resolved_indexes AS |
|
||||||
SELECT s.id AS scope_id, m.id AS master_index_id, a.archive_id, c.data, c.id AS container_id |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN master_indexes m |
|
||||||
JOIN master_index_archives a ON a.master_index_id = m.id |
|
||||||
JOIN resolve_index(s.id, a.archive_id, a.crc32, a.version) c ON TRUE; |
|
||||||
|
|
||||||
CREATE VIEW resolved_groups (scope_id, master_index_id, archive_id, group_id, name_hash, version, data, encrypted, empty_loc, key_id) AS |
|
||||||
WITH i AS NOT MATERIALIZED ( |
|
||||||
SELECT scope_id, master_index_id, archive_id, data, container_id |
|
||||||
FROM resolved_indexes |
|
||||||
) |
|
||||||
SELECT i.scope_id, i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, FALSE, FALSE, NULL |
|
||||||
FROM i |
|
||||||
UNION ALL |
|
||||||
SELECT i.scope_id, i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.encrypted, c.empty_loc, c.key_id |
|
||||||
FROM i |
|
||||||
JOIN index_groups ig ON ig.container_id = i.container_id |
|
||||||
JOIN resolve_group(i.scope_id, i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE; |
|
||||||
|
|
||||||
DROP VIEW colliding_groups; |
|
||||||
|
|
||||||
CREATE VIEW colliding_groups (scope_id, archive_id, group_id, crc32, truncated_version, versions, containers) AS |
|
||||||
SELECT |
|
||||||
g.scope_id, |
|
||||||
g.archive_id, |
|
||||||
g.group_id, |
|
||||||
c.crc32, |
|
||||||
g.version & 65535 AS truncated_version, |
|
||||||
array_agg(DISTINCT g.version ORDER BY g.version ASC), |
|
||||||
array_agg(DISTINCT c.id ORDER BY c.id ASC) |
|
||||||
FROM groups g |
|
||||||
JOIN containers c ON c.id = g.container_id |
|
||||||
GROUP BY g.scope_id, g.archive_id, g.group_id, c.crc32, truncated_version |
|
||||||
HAVING COUNT(DISTINCT c.id) > 1; |
|
||||||
|
|
||||||
DROP VIEW cache_stats; |
|
||||||
DROP MATERIALIZED VIEW master_index_stats; |
|
||||||
DROP MATERIALIZED VIEW index_stats; |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW index_stats ( |
|
||||||
scope_id, |
|
||||||
archive_id, |
|
||||||
container_id, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
g.group_id AS archive_id, |
|
||||||
i.container_id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups, |
|
||||||
COUNT(*) AS groups, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted) AS keys, |
|
||||||
SUM(length(c.data) + 2) FILTER (WHERE c.id IS NOT NULL) AS size, |
|
||||||
SUM(group_blocks(ig.group_id, length(c.data) + 2)) FILTER (WHERE c.id IS NOT NULL) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN indexes i |
|
||||||
JOIN groups g ON g.container_id = i.container_id AND g.archive_id = 255 AND NOT g.version_truncated AND |
|
||||||
g.version = i.version |
|
||||||
JOIN index_groups ig ON ig.container_id = i.container_id |
|
||||||
LEFT JOIN resolve_group(s.id, g.group_id::uint1, ig.group_id, ig.crc32, ig.version) c ON TRUE |
|
||||||
GROUP BY s.id, g.group_id, i.container_id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON index_stats (scope_id, archive_id, container_id); |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW master_index_stats ( |
|
||||||
scope_id, |
|
||||||
master_index_id, |
|
||||||
valid_indexes, |
|
||||||
indexes, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
sc.id, |
|
||||||
m.id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, |
|
||||||
COUNT(*) FILTER (WHERE a.master_index_id IS NOT NULL) AS indexes, |
|
||||||
SUM(COALESCE(s.valid_groups, 0)) AS valid_groups, |
|
||||||
SUM(COALESCE(s.groups, 0)) AS groups, |
|
||||||
SUM(COALESCE(s.valid_keys, 0)) AS valid_keys, |
|
||||||
SUM(COALESCE(s.keys, 0)) AS keys, |
|
||||||
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(c.data), 0)) AS size, |
|
||||||
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(c.data)), 0)) AS blocks |
|
||||||
FROM scopes sc |
|
||||||
CROSS JOIN master_indexes m |
|
||||||
LEFT JOIN master_index_archives a ON a.master_index_id = m.id |
|
||||||
LEFT JOIN resolve_index(sc.id, a.archive_id, a.crc32, a.version) c ON TRUE |
|
||||||
LEFT JOIN index_stats s ON s.scope_id = sc.id AND s.archive_id = a.archive_id AND s.container_id = c.id |
|
||||||
GROUP BY sc.id, m.id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON master_index_stats (scope_id, master_index_id); |
|
||||||
|
|
||||||
CREATE VIEW cache_stats AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
c.id AS cache_id, |
|
||||||
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes, |
|
||||||
COALESCE(ms.indexes, cs.archives) AS indexes, |
|
||||||
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups, |
|
||||||
COALESCE(ms.groups, cs.files) AS groups, |
|
||||||
COALESCE(ms.valid_keys, 0) AS valid_keys, |
|
||||||
COALESCE(ms.keys, 0) AS keys, |
|
||||||
COALESCE(ms.size, cs.size) AS size, |
|
||||||
COALESCE(ms.blocks, cs.blocks) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN caches c |
|
||||||
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id |
|
||||||
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id; |
|
||||||
|
|
||||||
DROP FUNCTION resolve_group(_archive_id uint1, _group_id INTEGER, _crc32 INTEGER, _version INTEGER); |
|
||||||
DROP FUNCTION resolve_index(_archive_id uint1, _crc32 INTEGER, _version INTEGER); |
|
||||||
@ -1,2 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
ALTER TABLE caches ADD COLUMN hidden BOOLEAN NOT NULL DEFAULT FALSE; |
|
||||||
@ -1,95 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
CREATE MATERIALIZED VIEW index_stats_new ( |
|
||||||
scope_id, |
|
||||||
archive_id, |
|
||||||
container_id, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
g.group_id AS archive_id, |
|
||||||
i.container_id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups, |
|
||||||
COUNT(*) AS groups, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted) AS keys, |
|
||||||
SUM(length(c.data) + 2) FILTER (WHERE c.id IS NOT NULL) AS size, |
|
||||||
SUM(group_blocks(ig.group_id, length(c.data) + 2)) FILTER (WHERE c.id IS NOT NULL) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN indexes i |
|
||||||
JOIN groups g ON g.scope_id = s.id AND g.container_id = i.container_id AND g.archive_id = 255 AND |
|
||||||
NOT g.version_truncated AND g.version = i.version |
|
||||||
JOIN index_groups ig ON ig.container_id = i.container_id |
|
||||||
LEFT JOIN resolve_group(s.id, g.group_id::uint1, ig.group_id, ig.crc32, ig.version) c ON TRUE |
|
||||||
GROUP BY s.id, g.group_id, i.container_id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON index_stats_new (scope_id, archive_id, container_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats RENAME TO index_stats_old; |
|
||||||
ALTER INDEX index_stats_scope_id_archive_id_container_id_idx RENAME TO index_stats_old_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats_new RENAME TO index_stats; |
|
||||||
ALTER INDEX index_stats_new_scope_id_archive_id_container_id_idx RENAME TO index_stats_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW master_index_stats_new ( |
|
||||||
scope_id, |
|
||||||
master_index_id, |
|
||||||
valid_indexes, |
|
||||||
indexes, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
sc.id, |
|
||||||
m.id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, |
|
||||||
COUNT(*) FILTER (WHERE a.master_index_id IS NOT NULL) AS indexes, |
|
||||||
SUM(COALESCE(s.valid_groups, 0)) AS valid_groups, |
|
||||||
SUM(COALESCE(s.groups, 0)) AS groups, |
|
||||||
SUM(COALESCE(s.valid_keys, 0)) AS valid_keys, |
|
||||||
SUM(COALESCE(s.keys, 0)) AS keys, |
|
||||||
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(c.data), 0)) AS size, |
|
||||||
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(c.data)), 0)) AS blocks |
|
||||||
FROM scopes sc |
|
||||||
CROSS JOIN master_indexes m |
|
||||||
LEFT JOIN master_index_archives a ON a.master_index_id = m.id |
|
||||||
LEFT JOIN resolve_index(sc.id, a.archive_id, a.crc32, a.version) c ON TRUE |
|
||||||
LEFT JOIN index_stats s ON s.scope_id = sc.id AND s.archive_id = a.archive_id AND s.container_id = c.id |
|
||||||
GROUP BY sc.id, m.id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON master_index_stats_new (scope_id, master_index_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats RENAME TO master_index_stats_old; |
|
||||||
ALTER INDEX master_index_stats_scope_id_master_index_id_idx RENAME TO master_index_stats_old_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats_new RENAME TO master_index_stats; |
|
||||||
ALTER INDEX master_index_stats_new_scope_id_master_index_id_idx RENAME TO master_index_stats_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
CREATE OR REPLACE VIEW cache_stats AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
c.id AS cache_id, |
|
||||||
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes, |
|
||||||
COALESCE(ms.indexes, cs.archives) AS indexes, |
|
||||||
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups, |
|
||||||
COALESCE(ms.groups, cs.files) AS groups, |
|
||||||
COALESCE(ms.valid_keys, 0) AS valid_keys, |
|
||||||
COALESCE(ms.keys, 0) AS keys, |
|
||||||
COALESCE(ms.size, cs.size) AS size, |
|
||||||
COALESCE(ms.blocks, cs.blocks) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN caches c |
|
||||||
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id |
|
||||||
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id; |
|
||||||
|
|
||||||
DROP MATERIALIZED VIEW master_index_stats_old; |
|
||||||
DROP MATERIALIZED VIEW index_stats_old; |
|
||||||
@ -1,95 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
CREATE MATERIALIZED VIEW index_stats_new ( |
|
||||||
scope_id, |
|
||||||
archive_id, |
|
||||||
container_id, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
g.group_id AS archive_id, |
|
||||||
i.container_id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups, |
|
||||||
COUNT(*) FILTER (WHERE ig.container_id IS NOT NULL) AS groups, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted) AS keys, |
|
||||||
SUM(length(c.data) + 2) FILTER (WHERE c.id IS NOT NULL) AS size, |
|
||||||
SUM(group_blocks(ig.group_id, length(c.data) + 2)) FILTER (WHERE c.id IS NOT NULL) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN indexes i |
|
||||||
JOIN groups g ON g.scope_id = s.id AND g.container_id = i.container_id AND g.archive_id = 255 AND |
|
||||||
NOT g.version_truncated AND g.version = i.version |
|
||||||
LEFT JOIN index_groups ig ON ig.container_id = i.container_id |
|
||||||
LEFT JOIN resolve_group(s.id, g.group_id::uint1, ig.group_id, ig.crc32, ig.version) c ON TRUE |
|
||||||
GROUP BY s.id, g.group_id, i.container_id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON index_stats_new (scope_id, archive_id, container_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats RENAME TO index_stats_old; |
|
||||||
ALTER INDEX index_stats_scope_id_archive_id_container_id_idx RENAME TO index_stats_old_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats_new RENAME TO index_stats; |
|
||||||
ALTER INDEX index_stats_new_scope_id_archive_id_container_id_idx RENAME TO index_stats_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW master_index_stats_new ( |
|
||||||
scope_id, |
|
||||||
master_index_id, |
|
||||||
valid_indexes, |
|
||||||
indexes, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
sc.id, |
|
||||||
m.id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, |
|
||||||
COUNT(*) FILTER (WHERE a.master_index_id IS NOT NULL) AS indexes, |
|
||||||
SUM(COALESCE(s.valid_groups, 0)) AS valid_groups, |
|
||||||
SUM(COALESCE(s.groups, 0)) AS groups, |
|
||||||
SUM(COALESCE(s.valid_keys, 0)) AS valid_keys, |
|
||||||
SUM(COALESCE(s.keys, 0)) AS keys, |
|
||||||
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(c.data), 0)) AS size, |
|
||||||
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(c.data)), 0)) AS blocks |
|
||||||
FROM scopes sc |
|
||||||
CROSS JOIN master_indexes m |
|
||||||
LEFT JOIN master_index_archives a ON a.master_index_id = m.id |
|
||||||
LEFT JOIN resolve_index(sc.id, a.archive_id, a.crc32, a.version) c ON TRUE |
|
||||||
LEFT JOIN index_stats s ON s.scope_id = sc.id AND s.archive_id = a.archive_id AND s.container_id = c.id |
|
||||||
GROUP BY sc.id, m.id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON master_index_stats_new (scope_id, master_index_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats RENAME TO master_index_stats_old; |
|
||||||
ALTER INDEX master_index_stats_scope_id_master_index_id_idx RENAME TO master_index_stats_old_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats_new RENAME TO master_index_stats; |
|
||||||
ALTER INDEX master_index_stats_new_scope_id_master_index_id_idx RENAME TO master_index_stats_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
CREATE OR REPLACE VIEW cache_stats AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
c.id AS cache_id, |
|
||||||
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes, |
|
||||||
COALESCE(ms.indexes, cs.archives) AS indexes, |
|
||||||
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups, |
|
||||||
COALESCE(ms.groups, cs.files) AS groups, |
|
||||||
COALESCE(ms.valid_keys, 0) AS valid_keys, |
|
||||||
COALESCE(ms.keys, 0) AS keys, |
|
||||||
COALESCE(ms.size, cs.size) AS size, |
|
||||||
COALESCE(ms.blocks, cs.blocks) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN caches c |
|
||||||
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id |
|
||||||
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id; |
|
||||||
|
|
||||||
DROP MATERIALIZED VIEW master_index_stats_old; |
|
||||||
DROP MATERIALIZED VIEW index_stats_old; |
|
||||||
@ -1,95 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
CREATE MATERIALIZED VIEW index_stats_new ( |
|
||||||
scope_id, |
|
||||||
archive_id, |
|
||||||
container_id, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
g.group_id AS archive_id, |
|
||||||
i.container_id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups, |
|
||||||
COUNT(*) FILTER (WHERE ig.container_id IS NOT NULL) AS groups, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys, |
|
||||||
COUNT(*) FILTER (WHERE c.encrypted) AS keys, |
|
||||||
COALESCE(SUM(length(c.data) + 2) FILTER (WHERE c.id IS NOT NULL), 0) AS size, |
|
||||||
COALESCE(SUM(group_blocks(ig.group_id, length(c.data) + 2)) FILTER (WHERE c.id IS NOT NULL), 0) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN indexes i |
|
||||||
JOIN groups g ON g.scope_id = s.id AND g.container_id = i.container_id AND g.archive_id = 255 AND |
|
||||||
NOT g.version_truncated AND g.version = i.version |
|
||||||
LEFT JOIN index_groups ig ON ig.container_id = i.container_id |
|
||||||
LEFT JOIN resolve_group(s.id, g.group_id::uint1, ig.group_id, ig.crc32, ig.version) c ON TRUE |
|
||||||
GROUP BY s.id, g.group_id, i.container_id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON index_stats_new (scope_id, archive_id, container_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats RENAME TO index_stats_old; |
|
||||||
ALTER INDEX index_stats_scope_id_archive_id_container_id_idx RENAME TO index_stats_old_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW index_stats_new RENAME TO index_stats; |
|
||||||
ALTER INDEX index_stats_new_scope_id_archive_id_container_id_idx RENAME TO index_stats_scope_id_archive_id_container_id_idx; |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW master_index_stats_new ( |
|
||||||
scope_id, |
|
||||||
master_index_id, |
|
||||||
valid_indexes, |
|
||||||
indexes, |
|
||||||
valid_groups, |
|
||||||
groups, |
|
||||||
valid_keys, |
|
||||||
keys, |
|
||||||
size, |
|
||||||
blocks |
|
||||||
) AS |
|
||||||
SELECT |
|
||||||
sc.id, |
|
||||||
m.id, |
|
||||||
COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, |
|
||||||
COUNT(*) FILTER (WHERE a.master_index_id IS NOT NULL) AS indexes, |
|
||||||
SUM(COALESCE(s.valid_groups, 0)) AS valid_groups, |
|
||||||
SUM(COALESCE(s.groups, 0)) AS groups, |
|
||||||
SUM(COALESCE(s.valid_keys, 0)) AS valid_keys, |
|
||||||
SUM(COALESCE(s.keys, 0)) AS keys, |
|
||||||
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(c.data), 0)) AS size, |
|
||||||
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(c.data)), 0)) AS blocks |
|
||||||
FROM scopes sc |
|
||||||
CROSS JOIN master_indexes m |
|
||||||
LEFT JOIN master_index_archives a ON a.master_index_id = m.id |
|
||||||
LEFT JOIN resolve_index(sc.id, a.archive_id, a.crc32, a.version) c ON TRUE |
|
||||||
LEFT JOIN index_stats s ON s.scope_id = sc.id AND s.archive_id = a.archive_id AND s.container_id = c.id |
|
||||||
GROUP BY sc.id, m.id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON master_index_stats_new (scope_id, master_index_id); |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats RENAME TO master_index_stats_old; |
|
||||||
ALTER INDEX master_index_stats_scope_id_master_index_id_idx RENAME TO master_index_stats_old_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
ALTER MATERIALIZED VIEW master_index_stats_new RENAME TO master_index_stats; |
|
||||||
ALTER INDEX master_index_stats_new_scope_id_master_index_id_idx RENAME TO master_index_stats_scope_id_master_index_id_idx; |
|
||||||
|
|
||||||
CREATE OR REPLACE VIEW cache_stats AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
c.id AS cache_id, |
|
||||||
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes, |
|
||||||
COALESCE(ms.indexes, cs.archives) AS indexes, |
|
||||||
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups, |
|
||||||
COALESCE(ms.groups, cs.files) AS groups, |
|
||||||
COALESCE(ms.valid_keys, 0) AS valid_keys, |
|
||||||
COALESCE(ms.keys, 0) AS keys, |
|
||||||
COALESCE(ms.size, cs.size) AS size, |
|
||||||
COALESCE(ms.blocks, cs.blocks) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN caches c |
|
||||||
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id |
|
||||||
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id; |
|
||||||
|
|
||||||
DROP MATERIALIZED VIEW master_index_stats_old; |
|
||||||
DROP MATERIALIZED VIEW index_stats_old; |
|
||||||
@ -1,53 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
DROP VIEW cache_stats; |
|
||||||
DROP MATERIALIZED VIEW crc_table_stats; |
|
||||||
DROP MATERIALIZED VIEW version_list_stats; |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW version_list_stats AS |
|
||||||
SELECT |
|
||||||
v.blob_id, |
|
||||||
vf.index_id, |
|
||||||
COUNT(*) FILTER (WHERE b.id IS NOT NULL) AS valid_files, |
|
||||||
COUNT(*) AS files, |
|
||||||
SUM(length(b.data) + 2) FILTER (WHERE b.id IS NOT NULL) AS size, |
|
||||||
SUM(group_blocks(vf.file_id, length(b.data) + 2)) AS blocks |
|
||||||
FROM version_lists v |
|
||||||
JOIN version_list_files vf ON vf.blob_id = v.blob_id |
|
||||||
LEFT JOIN resolve_file(vf.index_id, vf.file_id, vf.version, vf.crc32) b ON TRUE |
|
||||||
GROUP BY v.blob_id, vf.index_id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON version_list_stats (blob_id, index_id); |
|
||||||
|
|
||||||
CREATE MATERIALIZED VIEW crc_table_stats AS |
|
||||||
SELECT |
|
||||||
c.id AS crc_table_id, |
|
||||||
COUNT(*) FILTER (WHERE b.id IS NOT NULL AND a.crc32 <> 0) AS valid_archives, |
|
||||||
COUNT(*) FILTER (WHERE a.crc32 <> 0) AS archives, |
|
||||||
SUM(COALESCE(s.valid_files, 0)) AS valid_files, |
|
||||||
SUM(COALESCE(s.files, 0)) AS files, |
|
||||||
SUM(COALESCE(s.size, 0)) + SUM(COALESCE(length(b.data), 0)) AS size, |
|
||||||
SUM(COALESCE(s.blocks, 0)) + SUM(COALESCE(group_blocks(a.archive_id, length(b.data)), 0)) AS blocks |
|
||||||
FROM crc_tables c |
|
||||||
LEFT JOIN crc_table_archives a ON a.crc_table_id = c.id |
|
||||||
LEFT JOIN resolve_archive(a.archive_id, a.crc32) b ON TRUE |
|
||||||
LEFT JOIN version_list_stats s ON s.blob_id = b.id |
|
||||||
GROUP BY c.id; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON crc_table_stats (crc_table_id); |
|
||||||
|
|
||||||
CREATE VIEW cache_stats AS |
|
||||||
SELECT |
|
||||||
s.id AS scope_id, |
|
||||||
c.id AS cache_id, |
|
||||||
COALESCE(ms.valid_indexes, cs.valid_archives) AS valid_indexes, |
|
||||||
COALESCE(ms.indexes, cs.archives) AS indexes, |
|
||||||
COALESCE(ms.valid_groups, cs.valid_files) AS valid_groups, |
|
||||||
COALESCE(ms.groups, cs.files) AS groups, |
|
||||||
COALESCE(ms.valid_keys, 0) AS valid_keys, |
|
||||||
COALESCE(ms.keys, 0) AS keys, |
|
||||||
COALESCE(ms.size, cs.size) AS size, |
|
||||||
COALESCE(ms.blocks, cs.blocks) AS blocks |
|
||||||
FROM scopes s |
|
||||||
CROSS JOIN caches c |
|
||||||
LEFT JOIN master_index_stats ms ON ms.scope_id = s.id AND ms.master_index_id = c.id |
|
||||||
LEFT JOIN crc_table_stats cs ON s.name = 'runescape' AND cs.crc_table_id = c.id; |
|
||||||
@ -1,2 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
ALTER TYPE key_source ADD VALUE 'hdos'; |
|
||||||
@ -1,3 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
ALTER TYPE source_type ADD VALUE 'cross_pollination'; |
|
||||||
@ -1,7 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
ALTER TABLE sources |
|
||||||
ALTER COLUMN cache_id DROP NOT NULL, |
|
||||||
ALTER COLUMN game_id DROP NOT NULL; |
|
||||||
|
|
||||||
CREATE UNIQUE INDEX ON sources (type) WHERE type = 'cross_pollination'; |
|
||||||
@ -1,3 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
ALTER TYPE source_type ADD VALUE 'manual'; |
|
||||||
@ -1,95 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
CREATE EXTENSION IF NOT EXISTS pgcrypto; |
|
||||||
|
|
||||||
ALTER TABLE blobs ADD COLUMN sha1 BYTEA NULL; |
|
||||||
|
|
||||||
UPDATE blobs SET sha1 = digest(data, 'sha1'); |
|
||||||
|
|
||||||
ALTER TABLE blobs ALTER COLUMN sha1 SET NOT NULL; |
|
||||||
|
|
||||||
-- not UNIQUE as SHA-1 collisions are possible |
|
||||||
CREATE INDEX ON blobs USING HASH (sha1); |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('shared'); |
|
||||||
INSERT INTO games (name, scope_id) VALUES ('shared', (SELECT id FROM scopes WHERE name = 'shared')); |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('classic'); |
|
||||||
INSERT INTO games (name, scope_id) VALUES ('classic', (SELECT id FROM scopes WHERE name = 'classic')); |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('mapview'); |
|
||||||
INSERT INTO games (name, scope_id) VALUES ('mapview', (SELECT id FROM scopes WHERE name = 'mapview')); |
|
||||||
|
|
||||||
CREATE TYPE artifact_type AS ENUM ( |
|
||||||
'browsercontrol', |
|
||||||
'client', |
|
||||||
'client_gl', |
|
||||||
'gluegen_rt', |
|
||||||
'jaggl', |
|
||||||
'jaggl_dri', |
|
||||||
'jagmisc', |
|
||||||
'jogl', |
|
||||||
'jogl_awt', |
|
||||||
'loader', |
|
||||||
'loader_gl', |
|
||||||
'unpackclass' |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TYPE artifact_format AS ENUM ( |
|
||||||
'cab', |
|
||||||
'jar', |
|
||||||
'native', |
|
||||||
'pack200', |
|
||||||
'packclass' |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TYPE os AS ENUM ( |
|
||||||
'independent', |
|
||||||
'windows', |
|
||||||
'macos', |
|
||||||
'linux', |
|
||||||
'solaris' |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TYPE arch AS ENUM ( |
|
||||||
'independent', |
|
||||||
'universal', |
|
||||||
'x86', |
|
||||||
'amd64', |
|
||||||
'powerpc', |
|
||||||
'sparc', |
|
||||||
'sparcv9' |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TYPE jvm AS ENUM ( |
|
||||||
'independent', |
|
||||||
'sun', |
|
||||||
'microsoft' |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TABLE artifacts ( |
|
||||||
blob_id BIGINT PRIMARY KEY NOT NULL REFERENCES blobs (id), |
|
||||||
game_id INTEGER NOT NULL REFERENCES games (id), |
|
||||||
environment_id INTEGER NOT NULL REFERENCES environments (id), |
|
||||||
build_major INTEGER NULL, |
|
||||||
build_minor INTEGER NULL, |
|
||||||
timestamp TIMESTAMPTZ NULL, |
|
||||||
type artifact_type NOT NULL, |
|
||||||
format artifact_format NOT NULL, |
|
||||||
os os NOT NULL, |
|
||||||
arch arch NOT NULL, |
|
||||||
jvm jvm NOT NULL |
|
||||||
); |
|
||||||
|
|
||||||
CREATE TABLE artifact_links ( |
|
||||||
blob_id BIGINT NOT NULL REFERENCES artifacts (blob_id), |
|
||||||
type artifact_type NOT NULL, |
|
||||||
format artifact_format NOT NULL, |
|
||||||
os os NOT NULL, |
|
||||||
arch arch NOT NULL, |
|
||||||
jvm jvm NOT NULL, |
|
||||||
sha1 BYTEA NOT NULL, |
|
||||||
crc32 INTEGER NULL, |
|
||||||
size INTEGER NULL, |
|
||||||
PRIMARY KEY (blob_id, type, format, os, arch, jvm) |
|
||||||
); |
|
||||||
@ -1,11 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
CREATE TABLE artifact_sources ( |
|
||||||
id SERIAL PRIMARY KEY NOT NULL, |
|
||||||
blob_id BIGINT NOT NULL REFERENCES artifacts (blob_id), |
|
||||||
name TEXT NULL, |
|
||||||
description TEXT NULL, |
|
||||||
url TEXT NULL |
|
||||||
); |
|
||||||
|
|
||||||
CREATE INDEX ON artifact_sources (blob_id); |
|
||||||
@ -1,7 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('loginapplet'); |
|
||||||
INSERT INTO games (name, scope_id) VALUES ('loginapplet', (SELECT id FROM scopes WHERE name = 'loginapplet')); |
|
||||||
|
|
||||||
INSERT INTO scopes (name) VALUES ('passapplet'); |
|
||||||
INSERT INTO games (name, scope_id) VALUES ('passapplet', (SELECT id FROM scopes WHERE name = 'passapplet')); |
|
||||||
@ -1,5 +0,0 @@ |
|||||||
-- @formatter:off |
|
||||||
|
|
||||||
ALTER TABLE artifact_sources |
|
||||||
ADD COLUMN file_name TEXT NULL, |
|
||||||
ADD COLUMN timestamp TIMESTAMPTZ NULL; |
|
||||||
@ -1,3 +1,2 @@ |
|||||||
-- @formatter:off |
|
||||||
ALTER TABLE games |
ALTER TABLE games |
||||||
DROP COLUMN key; |
DROP COLUMN key; |
||||||
|
|||||||
@ -1,43 +0,0 @@ |
|||||||
var buildRegex = new RegExp('>([0-9]+)(?:[.]([0-9]+))?<'); |
|
||||||
|
|
||||||
function customSort(name, order, data) { |
|
||||||
order = order === 'asc' ? 1 : -1; |
|
||||||
|
|
||||||
data.sort(function (a, b) { |
|
||||||
a = a[name]; |
|
||||||
b = b[name]; |
|
||||||
|
|
||||||
if (!a) { |
|
||||||
return 1; |
|
||||||
} else if (!b) { |
|
||||||
return -1; |
|
||||||
} |
|
||||||
|
|
||||||
if (name === 'builds') { |
|
||||||
return buildSort(a, b) * order; |
|
||||||
} else { |
|
||||||
if (a < b) { |
|
||||||
return -order; |
|
||||||
} else if (a === b) { |
|
||||||
return 0; |
|
||||||
} else { |
|
||||||
return order; |
|
||||||
} |
|
||||||
} |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
function buildSort(a, b) { |
|
||||||
a = buildRegex.exec(a); |
|
||||||
b = buildRegex.exec(b); |
|
||||||
|
|
||||||
var aMajor = parseInt(a[1]); |
|
||||||
var bMajor = parseInt(b[1]); |
|
||||||
if (aMajor !== bMajor) { |
|
||||||
return aMajor - bMajor; |
|
||||||
} |
|
||||||
|
|
||||||
var aMinor = a[2] ? parseInt(a[2]) : 0; |
|
||||||
var bMinor = b[2] ? parseInt(b[2]) : 0; |
|
||||||
return aMinor - bMinor; |
|
||||||
} |
|
||||||
@ -1,285 +0,0 @@ |
|||||||
<!DOCTYPE html> |
|
||||||
<html xmlns:th="http://www.thymeleaf.org" lang="en"> |
|
||||||
<head th:replace="layout.html :: head"> |
|
||||||
<title>API - OpenRS2 Archive</title> |
|
||||||
<link rel="stylesheet" href="/webjars/bootstrap/css/bootstrap.min.css" /> |
|
||||||
<link rel="stylesheet" href="/static/css/openrs2.css" /> |
|
||||||
<script src="/webjars/jquery/jquery.min.js" defer></script> |
|
||||||
<script src="/webjars/bootstrap/js/bootstrap.bundle.min.js" defer></script> |
|
||||||
</head> |
|
||||||
<body> |
|
||||||
<nav th:replace="layout.html :: nav"></nav> |
|
||||||
<main class="container"> |
|
||||||
<h1>API</h1> |
|
||||||
|
|
||||||
<p>All endpoints accept requests from any origin. Range requests are not supported by any endpoint.</p> |
|
||||||
|
|
||||||
<h2><code>GET /caches.json</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a list of all caches, including all data available on the main <a href="/caches">caches</a> |
|
||||||
page, in JSON format: |
|
||||||
</p> |
|
||||||
|
|
||||||
<pre><code>[ |
|
||||||
{ |
|
||||||
// The cache's internal ID. |
|
||||||
"id": 1, |
|
||||||
|
|
||||||
// A scope is a group of related games. Missing groups are only located |
|
||||||
// from caches for games in the same scope. |
|
||||||
// |
|
||||||
// Currently the "runescape" scope is used for the "runescape" and |
|
||||||
// "oldschool" games. Each FunOrb game has its own scope. |
|
||||||
// |
|
||||||
// Your code must be prepared for new scopes to be added in the future. |
|
||||||
"scope": "runescape", |
|
||||||
|
|
||||||
// The game's name. Your code must be prepared for new games to be |
|
||||||
// added in the future. |
|
||||||
"game": "runescape", |
|
||||||
|
|
||||||
// Currently either "live" or "beta", but your code must be prepared |
|
||||||
// for new environments to be added in the future. |
|
||||||
"environment": "live", |
|
||||||
|
|
||||||
// The language's ISO-639-1 code. Currently either "en", "de", "fr" or |
|
||||||
// "pt", but your code must be prepared for new languages to be added |
|
||||||
// in the future. |
|
||||||
"language": "en", |
|
||||||
|
|
||||||
// A list of build numbers the cache is associated with, which may be |
|
||||||
// empty if the build number(s) are not known. |
|
||||||
"builds": [ |
|
||||||
{ |
|
||||||
// The major number is always set. |
|
||||||
"major": 549, |
|
||||||
|
|
||||||
// The minor number may be null. |
|
||||||
"minor": null |
|
||||||
}, |
|
||||||
{ |
|
||||||
"major": 550, |
|
||||||
"minor": null |
|
||||||
} |
|
||||||
], |
|
||||||
|
|
||||||
// The earliest timestamp the cache was available to users, in ISO 8601 |
|
||||||
// format. May be null if not known. |
|
||||||
"timestamp": "2009-06-12T14:55:58Z", |
|
||||||
|
|
||||||
// A list of users who provided a copy of this cache. |
|
||||||
// |
|
||||||
// May be empty if the users wished to remain anonymous. |
|
||||||
// |
|
||||||
// The value "Jagex" indicates the cache was directly downloaded from |
|
||||||
// Jagex's servers by the OpenRS2 project, so we are completely certain |
|
||||||
// it is genuine. This value will never be used for a cache obtained |
|
||||||
// from a third party. |
|
||||||
"sources": [ |
|
||||||
"Erand", |
|
||||||
"Hlwys", |
|
||||||
"Jagex", |
|
||||||
"K4rn4ge", |
|
||||||
"Nathan", |
|
||||||
"Rune-Wars" |
|
||||||
], |
|
||||||
|
|
||||||
// In old engine caches, the number of valid .jag archives that are not |
|
||||||
// missing. |
|
||||||
// |
|
||||||
// In new engine caches, the number of valid JS5 indexes that are not |
|
||||||
// missing. |
|
||||||
// |
|
||||||
// May be null if the cache is still being processed. |
|
||||||
"valid_indexes": 29, |
|
||||||
|
|
||||||
// In old engine caches, the total number of .jag archives that should |
|
||||||
// exist, based on the cache's CRC table. |
|
||||||
// |
|
||||||
// In new engine caches, the total number of JS5 indexes that should |
|
||||||
// exist, based on the JS5 master index. |
|
||||||
// |
|
||||||
// May be null if the cache is still being processed. |
|
||||||
"indexes": 29, |
|
||||||
|
|
||||||
// The number of valid files (old engine) or valid groups (new engine) |
|
||||||
// that are not missing. May be null if the cache is still being processed. |
|
||||||
"valid_groups": 71002, |
|
||||||
|
|
||||||
// In old engine caches, the total number of files that should exist, |
|
||||||
// based on the cache's versionlist.jag archive. |
|
||||||
// |
|
||||||
// In new engine caches, the total number of groups that should exist, |
|
||||||
// based on the JS5 indexes that are available. |
|
||||||
// |
|
||||||
// May be null if the cache is still being processed. |
|
||||||
"groups": 71146, |
|
||||||
|
|
||||||
// The number of encrypted groups for which a valid key is available. |
|
||||||
// May be null if the cache is still being processed. |
|
||||||
"valid_keys": 1203, |
|
||||||
|
|
||||||
// The total number of encrypted groups in the cache. May be null if |
|
||||||
// the cache is still being processed. |
|
||||||
"keys": 1240, |
|
||||||
|
|
||||||
// The total size of all groups in the cache in bytes. May be null if |
|
||||||
// the cache is still being processed. |
|
||||||
"size": 74970573, |
|
||||||
|
|
||||||
// The number of 520-byte blocks required to store the cache's data in |
|
||||||
// a .dat2 file. May be null if the cache is still being processed. |
|
||||||
"blocks": 185273, |
|
||||||
|
|
||||||
// A boolean flag indicating if the cache is small enough to be |
|
||||||
// downloaded in .dat2/.idx format. May be null if the cache is still |
|
||||||
// being processed. |
|
||||||
"disk_store_valid": true |
|
||||||
}, |
|
||||||
... |
|
||||||
]</code></pre> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/disk.zip</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a cache as a ZIP archive of <code>.dat/.idx</code> |
|
||||||
(old engine) or <code>.dat2/.idx</code> (new engine) files. All |
|
||||||
files are stored underneath a <code>cache</code> subdirectory |
|
||||||
in the zip archive. |
|
||||||
</p> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/flat-file.tar.gz</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a cache as a gzipped tarball of files, where each |
|
||||||
file in the tarball holds a single file from the cache (old |
|
||||||
engine) or single group (new engine). |
|
||||||
</p> |
|
||||||
|
|
||||||
<p> |
|
||||||
The paths within the archive all have a format of |
|
||||||
<code>cache/<index>/<file>.dat</code> (old engine) |
|
||||||
or <code>cache/<archive>/<group>.dat</code> (new |
|
||||||
engine). |
|
||||||
</p> |
|
||||||
|
|
||||||
<p>The two byte version trailers are included.</p> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/keys.json</code></h2> |
|
||||||
|
|
||||||
<p>Returns a list of valid XTEA keys for the cache in JSON format:</p> |
|
||||||
|
|
||||||
<pre><code>[ |
|
||||||
{ |
|
||||||
// The ID of the archive containing the group the key is used for. |
|
||||||
// Typically this is 5 (maps), but do note that RuneScape 3 does |
|
||||||
// support encrypting interfaces, though the functionality has not yet |
|
||||||
// been used, and some FunOrb games also have encrypted groups. |
|
||||||
"archive": 5, |
|
||||||
|
|
||||||
// The ID of the group the key is used for. |
|
||||||
"group": 1, |
|
||||||
|
|
||||||
// The group's name hash, or null if the group has no name. |
|
||||||
"name_hash": -1153472937, |
|
||||||
|
|
||||||
// The name of the group, if available, or null if the group has no |
|
||||||
// name or if the name is not known. |
|
||||||
"name": "l40_55", |
|
||||||
|
|
||||||
// The ID of the map square, if the group is an encrypted loc group |
|
||||||
// (has a name of lX_Z). The map square ID is ((X << 8) | Z). |
|
||||||
// null if the group is not an encrypted loc group. |
|
||||||
"mapsquare": 10295, |
|
||||||
|
|
||||||
// The XTEA key, represented as four 32-bit integers. |
|
||||||
"key": [ |
|
||||||
-1920480496, |
|
||||||
-1423914110, |
|
||||||
951774544, |
|
||||||
-1419269290 |
|
||||||
] |
|
||||||
}, |
|
||||||
... |
|
||||||
]</code></pre> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/keys.zip</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a zip archive file of valid XTEA keys for loc groups. |
|
||||||
Each key is stored in a text file containing four lines, with |
|
||||||
each line containing a 32-bit component of the key as a decimal |
|
||||||
string. The paths within the archive all have a format of |
|
||||||
<code>keys/<mapsquare>.txt</code>. |
|
||||||
</p> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/map.png</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Renders the map squares in the cache, with a coloured outline |
|
||||||
representing whether we have a valid key for each map square or |
|
||||||
not: |
|
||||||
</p> |
|
||||||
|
|
||||||
<ul> |
|
||||||
<li><strong>Valid key:</strong> green outline.</li> |
|
||||||
<li><strong>Loc group is not encrypted:</strong> green outline.</li> |
|
||||||
<li><strong>Empty loc group:</strong> grey outline.</li> |
|
||||||
<li><strong>Key unknown:</strong> red outline.</li> |
|
||||||
</ul> |
|
||||||
|
|
||||||
<p> |
|
||||||
Empty loc groups may be replaced with an unencrypted equivalent |
|
||||||
with a cache editor. |
|
||||||
</p> |
|
||||||
|
|
||||||
<h2><code>GET /caches/<scope>/<id>/archives/<archive>/groups/<group>.dat</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a single file (old engine) or group (new engine) in |
|
||||||
binary format. The response contains a <code>.jag</code> |
|
||||||
archive (index 0 of an old engine cache), a GZIP-compressed |
|
||||||
file (the remaining indexes of an old engine cache) or |
|
||||||
JS5-compressed data (new engine cache, also known as a |
|
||||||
container). The two byte version trailer is not included. |
|
||||||
</p> |
|
||||||
|
|
||||||
<h2><code>GET /keys/all.json</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a list of all XTEA keys in the database, including |
|
||||||
candidate keys that have not been validated against any cache. |
|
||||||
</p> |
|
||||||
|
|
||||||
<pre><code>[ |
|
||||||
// The XTEA key, represented as four 32-bit integers. |
|
||||||
[ |
|
||||||
-2147135705, |
|
||||||
1113423446, |
|
||||||
1294100345, |
|
||||||
946019601 |
|
||||||
], |
|
||||||
... |
|
||||||
]</code></pre> |
|
||||||
|
|
||||||
<h2><code>GET /keys/valid.json</code></h2> |
|
||||||
|
|
||||||
<p> |
|
||||||
Returns a list of XTEA keys in the database, only including |
|
||||||
keys validated against at least one cache. |
|
||||||
</p> |
|
||||||
|
|
||||||
<pre><code>[ |
|
||||||
// The XTEA key, represented as four 32-bit integers. |
|
||||||
[ |
|
||||||
-2147135705, |
|
||||||
1113423446, |
|
||||||
1294100345, |
|
||||||
946019601 |
|
||||||
], |
|
||||||
... |
|
||||||
]</code></pre> |
|
||||||
</main> |
|
||||||
</body> |
|
||||||
</html> |
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue