forked from openrs2/openrs2
I have a small collection of these from when I ran a service that polled the JS5 server for the master index back in 2009. We'll probably be able to find some in old Rune-Server threads too. They'll be useful for verifying caches with an unclear provenance. Signed-off-by: Graham <gpe@openrs2.org>
parent
19b6893681
commit
30b605d719
@ -0,0 +1,16 @@ |
|||||||
|
package org.openrs2.archive.container |
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf |
||||||
|
import io.netty.buffer.ByteBufUtil |
||||||
|
import io.netty.buffer.DefaultByteBufHolder |
||||||
|
import org.openrs2.buffer.crc32 |
||||||
|
import org.openrs2.crypto.Whirlpool |
||||||
|
|
||||||
|
public abstract class Container( |
||||||
|
data: ByteBuf |
||||||
|
) : DefaultByteBufHolder(data) { |
||||||
|
public val bytes: ByteArray = ByteBufUtil.getBytes(data, data.readerIndex(), data.readableBytes(), false) |
||||||
|
public val crc32: Int = data.crc32() |
||||||
|
public val whirlpool: ByteArray = Whirlpool.whirlpool(bytes) |
||||||
|
public abstract val encrypted: Boolean |
||||||
|
} |
@ -0,0 +1,94 @@ |
|||||||
|
package org.openrs2.archive.container |
||||||
|
|
||||||
|
import java.sql.Connection |
||||||
|
|
||||||
|
public object ContainerImporter { |
||||||
|
public fun prepare(connection: Connection) { |
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
LOCK TABLE containers IN EXCLUSIVE MODE |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.execute() |
||||||
|
} |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
CREATE TEMPORARY TABLE tmp_containers ( |
||||||
|
index INTEGER NOT NULL, |
||||||
|
crc32 INTEGER NOT NULL, |
||||||
|
whirlpool BYTEA NOT NULL, |
||||||
|
data BYTEA NOT NULL, |
||||||
|
encrypted BOOLEAN NOT NULL |
||||||
|
) ON COMMIT DROP |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.execute() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public fun addContainer(connection: Connection, container: Container): Long { |
||||||
|
return addContainers(connection, listOf(container)).single() |
||||||
|
} |
||||||
|
|
||||||
|
public fun addContainers(connection: Connection, containers: List<Container>): List<Long> { |
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
TRUNCATE TABLE tmp_containers |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.execute() |
||||||
|
} |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
INSERT INTO tmp_containers (index, crc32, whirlpool, data, encrypted) |
||||||
|
VALUES (?, ?, ?, ?, ?) |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
for ((i, container) in containers.withIndex()) { |
||||||
|
stmt.setInt(1, i) |
||||||
|
stmt.setInt(2, container.crc32) |
||||||
|
stmt.setBytes(3, container.whirlpool) |
||||||
|
stmt.setBytes(4, container.bytes) |
||||||
|
stmt.setBoolean(5, container.encrypted) |
||||||
|
stmt.addBatch() |
||||||
|
} |
||||||
|
|
||||||
|
stmt.executeBatch() |
||||||
|
} |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
INSERT INTO containers (crc32, whirlpool, data, encrypted) |
||||||
|
SELECT t.crc32, t.whirlpool, t.data, t.encrypted |
||||||
|
FROM tmp_containers t |
||||||
|
LEFT JOIN containers c ON c.whirlpool = t.whirlpool |
||||||
|
WHERE c.whirlpool IS NULL |
||||||
|
ON CONFLICT DO NOTHING |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.execute() |
||||||
|
} |
||||||
|
|
||||||
|
val ids = mutableListOf<Long>() |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
SELECT c.id |
||||||
|
FROM tmp_containers t |
||||||
|
JOIN containers c ON c.whirlpool = t.whirlpool |
||||||
|
ORDER BY t.index ASC |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.executeQuery().use { rows -> |
||||||
|
while (rows.next()) { |
||||||
|
ids += rows.getLong(1) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
check(ids.size == containers.size) |
||||||
|
return ids |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,28 @@ |
|||||||
|
package org.openrs2.archive.masterindex |
||||||
|
|
||||||
|
import com.github.ajalt.clikt.core.CliktCommand |
||||||
|
import com.github.ajalt.clikt.parameters.arguments.argument |
||||||
|
import com.github.ajalt.clikt.parameters.types.path |
||||||
|
import com.google.inject.Guice |
||||||
|
import io.netty.buffer.Unpooled |
||||||
|
import kotlinx.coroutines.runBlocking |
||||||
|
import org.openrs2.archive.ArchiveModule |
||||||
|
import org.openrs2.buffer.use |
||||||
|
import java.nio.file.Files |
||||||
|
|
||||||
|
public class ImportCommand : CliktCommand(name = "import") { |
||||||
|
private val input by argument().path( |
||||||
|
mustExist = true, |
||||||
|
canBeDir = false, |
||||||
|
mustBeReadable = true |
||||||
|
) |
||||||
|
|
||||||
|
override fun run(): Unit = runBlocking { |
||||||
|
val injector = Guice.createInjector(ArchiveModule) |
||||||
|
val importer = injector.getInstance(MasterIndexImporter::class.java) |
||||||
|
|
||||||
|
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> |
||||||
|
importer.import(buf) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,12 @@ |
|||||||
|
package org.openrs2.archive.masterindex |
||||||
|
|
||||||
|
import com.github.ajalt.clikt.core.NoOpCliktCommand |
||||||
|
import com.github.ajalt.clikt.core.subcommands |
||||||
|
|
||||||
|
public class MasterIndexCommand : NoOpCliktCommand(name = "master-index") { |
||||||
|
init { |
||||||
|
subcommands( |
||||||
|
ImportCommand() |
||||||
|
) |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,63 @@ |
|||||||
|
package org.openrs2.archive.masterindex |
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf |
||||||
|
import org.openrs2.archive.container.Container |
||||||
|
import org.openrs2.archive.container.ContainerImporter |
||||||
|
import org.openrs2.buffer.use |
||||||
|
import org.openrs2.cache.Js5Compression |
||||||
|
import org.openrs2.cache.Js5MasterIndex |
||||||
|
import org.openrs2.db.Database |
||||||
|
import javax.inject.Inject |
||||||
|
import javax.inject.Singleton |
||||||
|
|
||||||
|
@Singleton |
||||||
|
public class MasterIndexImporter @Inject constructor( |
||||||
|
private val database: Database |
||||||
|
) { |
||||||
|
private class MasterIndex( |
||||||
|
data: ByteBuf, |
||||||
|
val index: Js5MasterIndex |
||||||
|
) : Container(data) { |
||||||
|
override val encrypted: Boolean = false |
||||||
|
} |
||||||
|
|
||||||
|
public suspend fun import(buf: ByteBuf) { |
||||||
|
database.execute { connection -> |
||||||
|
ContainerImporter.prepare(connection) |
||||||
|
|
||||||
|
val masterIndex = Js5Compression.uncompress(buf).use { uncompressed -> |
||||||
|
MasterIndex(buf, Js5MasterIndex.read(uncompressed)) |
||||||
|
} |
||||||
|
|
||||||
|
val containerId = ContainerImporter.addContainer(connection, masterIndex) |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
INSERT INTO master_indexes (container_id) |
||||||
|
VALUES (?) |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
stmt.setLong(1, containerId) |
||||||
|
stmt.execute() |
||||||
|
} |
||||||
|
|
||||||
|
connection.prepareStatement( |
||||||
|
""" |
||||||
|
INSERT INTO master_index_entries (container_id, archive_id, crc32, version) |
||||||
|
VALUES (?, ?, ?, ?) |
||||||
|
""".trimIndent() |
||||||
|
).use { stmt -> |
||||||
|
for ((i, entry) in masterIndex.index.entries.withIndex()) { |
||||||
|
stmt.setLong(1, containerId) |
||||||
|
stmt.setInt(2, i) |
||||||
|
stmt.setInt(3, entry.checksum) |
||||||
|
stmt.setInt(4, entry.version) |
||||||
|
|
||||||
|
stmt.addBatch() |
||||||
|
} |
||||||
|
|
||||||
|
stmt.executeBatch() |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
Loading…
Reference in new issue