I have a small collection of these from when I ran a service that polled the JS5 server for the master index back in 2009. We'll probably be able to find some in old Rune-Server threads too. They'll be useful for verifying caches with an unclear provenance. Signed-off-by: Graham <gpe@openrs2.org>pull/132/head
parent
19b6893681
commit
30b605d719
@ -0,0 +1,16 @@ |
||||
package org.openrs2.archive.container |
||||
|
||||
import io.netty.buffer.ByteBuf |
||||
import io.netty.buffer.ByteBufUtil |
||||
import io.netty.buffer.DefaultByteBufHolder |
||||
import org.openrs2.buffer.crc32 |
||||
import org.openrs2.crypto.Whirlpool |
||||
|
||||
public abstract class Container( |
||||
data: ByteBuf |
||||
) : DefaultByteBufHolder(data) { |
||||
public val bytes: ByteArray = ByteBufUtil.getBytes(data, data.readerIndex(), data.readableBytes(), false) |
||||
public val crc32: Int = data.crc32() |
||||
public val whirlpool: ByteArray = Whirlpool.whirlpool(bytes) |
||||
public abstract val encrypted: Boolean |
||||
} |
||||
@ -0,0 +1,94 @@ |
||||
package org.openrs2.archive.container |
||||
|
||||
import java.sql.Connection |
||||
|
||||
public object ContainerImporter { |
||||
public fun prepare(connection: Connection) { |
||||
connection.prepareStatement( |
||||
""" |
||||
LOCK TABLE containers IN EXCLUSIVE MODE |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.execute() |
||||
} |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
CREATE TEMPORARY TABLE tmp_containers ( |
||||
index INTEGER NOT NULL, |
||||
crc32 INTEGER NOT NULL, |
||||
whirlpool BYTEA NOT NULL, |
||||
data BYTEA NOT NULL, |
||||
encrypted BOOLEAN NOT NULL |
||||
) ON COMMIT DROP |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.execute() |
||||
} |
||||
} |
||||
|
||||
public fun addContainer(connection: Connection, container: Container): Long { |
||||
return addContainers(connection, listOf(container)).single() |
||||
} |
||||
|
||||
public fun addContainers(connection: Connection, containers: List<Container>): List<Long> { |
||||
connection.prepareStatement( |
||||
""" |
||||
TRUNCATE TABLE tmp_containers |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.execute() |
||||
} |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
INSERT INTO tmp_containers (index, crc32, whirlpool, data, encrypted) |
||||
VALUES (?, ?, ?, ?, ?) |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
for ((i, container) in containers.withIndex()) { |
||||
stmt.setInt(1, i) |
||||
stmt.setInt(2, container.crc32) |
||||
stmt.setBytes(3, container.whirlpool) |
||||
stmt.setBytes(4, container.bytes) |
||||
stmt.setBoolean(5, container.encrypted) |
||||
stmt.addBatch() |
||||
} |
||||
|
||||
stmt.executeBatch() |
||||
} |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
INSERT INTO containers (crc32, whirlpool, data, encrypted) |
||||
SELECT t.crc32, t.whirlpool, t.data, t.encrypted |
||||
FROM tmp_containers t |
||||
LEFT JOIN containers c ON c.whirlpool = t.whirlpool |
||||
WHERE c.whirlpool IS NULL |
||||
ON CONFLICT DO NOTHING |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.execute() |
||||
} |
||||
|
||||
val ids = mutableListOf<Long>() |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
SELECT c.id |
||||
FROM tmp_containers t |
||||
JOIN containers c ON c.whirlpool = t.whirlpool |
||||
ORDER BY t.index ASC |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.executeQuery().use { rows -> |
||||
while (rows.next()) { |
||||
ids += rows.getLong(1) |
||||
} |
||||
} |
||||
} |
||||
|
||||
check(ids.size == containers.size) |
||||
return ids |
||||
} |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
package org.openrs2.archive.masterindex |
||||
|
||||
import com.github.ajalt.clikt.core.CliktCommand |
||||
import com.github.ajalt.clikt.parameters.arguments.argument |
||||
import com.github.ajalt.clikt.parameters.types.path |
||||
import com.google.inject.Guice |
||||
import io.netty.buffer.Unpooled |
||||
import kotlinx.coroutines.runBlocking |
||||
import org.openrs2.archive.ArchiveModule |
||||
import org.openrs2.buffer.use |
||||
import java.nio.file.Files |
||||
|
||||
public class ImportCommand : CliktCommand(name = "import") { |
||||
private val input by argument().path( |
||||
mustExist = true, |
||||
canBeDir = false, |
||||
mustBeReadable = true |
||||
) |
||||
|
||||
override fun run(): Unit = runBlocking { |
||||
val injector = Guice.createInjector(ArchiveModule) |
||||
val importer = injector.getInstance(MasterIndexImporter::class.java) |
||||
|
||||
Unpooled.wrappedBuffer(Files.readAllBytes(input)).use { buf -> |
||||
importer.import(buf) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
package org.openrs2.archive.masterindex |
||||
|
||||
import com.github.ajalt.clikt.core.NoOpCliktCommand |
||||
import com.github.ajalt.clikt.core.subcommands |
||||
|
||||
public class MasterIndexCommand : NoOpCliktCommand(name = "master-index") { |
||||
init { |
||||
subcommands( |
||||
ImportCommand() |
||||
) |
||||
} |
||||
} |
||||
@ -0,0 +1,63 @@ |
||||
package org.openrs2.archive.masterindex |
||||
|
||||
import io.netty.buffer.ByteBuf |
||||
import org.openrs2.archive.container.Container |
||||
import org.openrs2.archive.container.ContainerImporter |
||||
import org.openrs2.buffer.use |
||||
import org.openrs2.cache.Js5Compression |
||||
import org.openrs2.cache.Js5MasterIndex |
||||
import org.openrs2.db.Database |
||||
import javax.inject.Inject |
||||
import javax.inject.Singleton |
||||
|
||||
@Singleton |
||||
public class MasterIndexImporter @Inject constructor( |
||||
private val database: Database |
||||
) { |
||||
private class MasterIndex( |
||||
data: ByteBuf, |
||||
val index: Js5MasterIndex |
||||
) : Container(data) { |
||||
override val encrypted: Boolean = false |
||||
} |
||||
|
||||
public suspend fun import(buf: ByteBuf) { |
||||
database.execute { connection -> |
||||
ContainerImporter.prepare(connection) |
||||
|
||||
val masterIndex = Js5Compression.uncompress(buf).use { uncompressed -> |
||||
MasterIndex(buf, Js5MasterIndex.read(uncompressed)) |
||||
} |
||||
|
||||
val containerId = ContainerImporter.addContainer(connection, masterIndex) |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
INSERT INTO master_indexes (container_id) |
||||
VALUES (?) |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
stmt.setLong(1, containerId) |
||||
stmt.execute() |
||||
} |
||||
|
||||
connection.prepareStatement( |
||||
""" |
||||
INSERT INTO master_index_entries (container_id, archive_id, crc32, version) |
||||
VALUES (?, ?, ?, ?) |
||||
""".trimIndent() |
||||
).use { stmt -> |
||||
for ((i, entry) in masterIndex.index.entries.withIndex()) { |
||||
stmt.setLong(1, containerId) |
||||
stmt.setInt(2, i) |
||||
stmt.setInt(3, entry.checksum) |
||||
stmt.setInt(4, entry.version) |
||||
|
||||
stmt.addBatch() |
||||
} |
||||
|
||||
stmt.executeBatch() |
||||
} |
||||
} |
||||
} |
||||
} |
||||
Loading…
Reference in new issue