From 39d2f18ccade6c0cf2a76fc39b630911f7bab668 Mon Sep 17 00:00:00 2001 From: Graham Date: Sun, 21 Aug 2022 15:24:28 +0100 Subject: [PATCH] Add tool for unpacking OpenNXT caches This is relatively easy as OpenNXT doesn't use the actual SQLite cache format - it still uses JS5-compressed containers, rather than ZLIB. Signed-off-by: Graham --- .../org/openrs2/cache/cli/CacheCommand.kt | 1 + .../openrs2/cache/cli/OpenNxtUnpackCommand.kt | 26 ++++++ cache/build.gradle.kts | 1 + .../kotlin/org/openrs2/cache/OpenNxtStore.kt | 85 +++++++++++++++++++ gradle/libs.versions.toml | 1 + 5 files changed, 114 insertions(+) create mode 100644 cache-cli/src/main/kotlin/org/openrs2/cache/cli/OpenNxtUnpackCommand.kt create mode 100644 cache/src/main/kotlin/org/openrs2/cache/OpenNxtStore.kt diff --git a/cache-cli/src/main/kotlin/org/openrs2/cache/cli/CacheCommand.kt b/cache-cli/src/main/kotlin/org/openrs2/cache/cli/CacheCommand.kt index d7a396d621..ef9ab16810 100644 --- a/cache-cli/src/main/kotlin/org/openrs2/cache/cli/CacheCommand.kt +++ b/cache-cli/src/main/kotlin/org/openrs2/cache/cli/CacheCommand.kt @@ -8,6 +8,7 @@ public fun main(args: Array): Unit = CacheCommand().main(args) public class CacheCommand : NoOpCliktCommand(name = "cache") { init { subcommands( + OpenNxtUnpackCommand(), RuneLiteUnpackCommand() ) } diff --git a/cache-cli/src/main/kotlin/org/openrs2/cache/cli/OpenNxtUnpackCommand.kt b/cache-cli/src/main/kotlin/org/openrs2/cache/cli/OpenNxtUnpackCommand.kt new file mode 100644 index 0000000000..f4a1475bf0 --- /dev/null +++ b/cache-cli/src/main/kotlin/org/openrs2/cache/cli/OpenNxtUnpackCommand.kt @@ -0,0 +1,26 @@ +package org.openrs2.cache.cli + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.types.path +import com.google.inject.Guice +import io.netty.buffer.ByteBufAllocator +import org.openrs2.cache.CacheModule +import org.openrs2.cache.OpenNxtStore +import org.openrs2.cache.Store +import org.openrs2.inject.CloseableInjector + +public class OpenNxtUnpackCommand : CliktCommand(name = "unpack-opennxt") { + private val input by argument().path(mustExist = true, canBeFile = false, mustBeReadable = true) + private val output by argument().path(canBeFile = false, mustBeReadable = true, mustBeWritable = true) + + override fun run() { + CloseableInjector(Guice.createInjector(CacheModule)).use { injector -> + val alloc = injector.getInstance(ByteBufAllocator::class.java) + + Store.open(output, alloc).use { store -> + OpenNxtStore.unpack(input, store) + } + } + } +} diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 3c4310e182..f76aa7afe0 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -13,6 +13,7 @@ dependencies { implementation(projects.buffer) implementation(projects.compress) implementation(projects.util) + implementation(libs.sqlite) testImplementation(libs.jimfs) } diff --git a/cache/src/main/kotlin/org/openrs2/cache/OpenNxtStore.kt b/cache/src/main/kotlin/org/openrs2/cache/OpenNxtStore.kt new file mode 100644 index 0000000000..0546ca2050 --- /dev/null +++ b/cache/src/main/kotlin/org/openrs2/cache/OpenNxtStore.kt @@ -0,0 +1,85 @@ +package org.openrs2.cache + +import io.netty.buffer.Unpooled +import org.openrs2.buffer.crc32 +import org.openrs2.buffer.use +import org.sqlite.SQLiteDataSource +import java.nio.file.Files +import java.nio.file.Path +import java.sql.Connection + +public object OpenNxtStore { + public fun unpack(input: Path, output: Store) { + output.create(Store.ARCHIVESET) + + for (archive in 0..Store.MAX_ARCHIVE) { + val path = input.resolve("js5-$archive.jcache") + if (!Files.exists(path)) { + continue + } + + val dataSource = SQLiteDataSource() + dataSource.url = "jdbc:sqlite:$path" + + dataSource.connection.use { connection -> + unpackArchive(connection, archive, output) + } + } + } + + private fun unpackArchive(connection: Connection, archive: Int, output: Store) { + connection.prepareStatement(""" + SELECT data, crc + FROM cache_index + WHERE key = 1 + """.trimIndent()).use { stmt -> + stmt.executeQuery().use { rows -> + if (rows.next()) { + val checksum = rows.getInt(2) + + Unpooled.wrappedBuffer(rows.getBytes(1)).use { buf -> + val actualChecksum = buf.crc32() + if (actualChecksum != checksum) { + throw StoreCorruptException( + "Js5Index corrupt (expected checksum $checksum, actual checksum $actualChecksum)" + ) + } + + output.write(Store.ARCHIVESET, archive, buf) + } + } + } + } + + connection.prepareStatement(""" + SELECT key, data, crc, version + FROM cache + """.trimIndent()).use { stmt -> + stmt.executeQuery().use { rows -> + while (rows.next()) { + val group = rows.getInt(1) + val checksum = rows.getInt(3) + val version = rows.getInt(4) and 0xFFFF + + Unpooled.wrappedBuffer(rows.getBytes(2)).use { buf -> + val actualVersion = VersionTrailer.peek(buf) + if (actualVersion != version) { + throw StoreCorruptException( + "Group corrupt (expected version $version, actual version $actualVersion)" + ) + } + + val actualChecksum = buf.slice(buf.readerIndex(), buf.writerIndex() - 2).crc32() + if (actualChecksum != checksum) { + throw StoreCorruptException( + "Group corrupt (expected checksum $checksum, actual checksum $actualChecksum)" + ) + } + + output.write(archive, group, buf) + } + } + } + } + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7b60c290ae..6651bcb4b6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -78,6 +78,7 @@ openrs2-natives = { module = "org.openrs2:openrs2-natives-all", version = "3.2.0 pf4j = { module = "org.pf4j:pf4j", version = "3.7.0" } postgres = { module = "org.postgresql:postgresql", version = "42.4.2" } runelite-client = { module = "net.runelite:client", version = "1.8.30" } +sqlite = { module = "org.xerial:sqlite-jdbc", version = "3.39.2.0" } thymeleaf-core = { module = "org.thymeleaf:thymeleaf", version = "3.0.15.RELEASE" } thymeleaf-java8time = { module = "org.thymeleaf.extras:thymeleaf-extras-java8time", version = "3.0.4.RELEASE" } xz = { module = "org.tukaani:xz", version = "1.9" }