From 416dabec4c0a6d86dea7d24503f8fd99c0db547c Mon Sep 17 00:00:00 2001 From: Graham Date: Sun, 14 Feb 2021 18:14:22 +0000 Subject: [PATCH] Include empty loc groups in the total number of known keys Signed-off-by: Graham --- .../openrs2/archive/cache/CacheImporter.kt | 19 +++-- .../org/openrs2/archive/V1__init.sql | 8 +- .../archive/templates/caches/index.html | 13 ++- .../org/openrs2/cache/Js5Compression.kt | 38 +++++++++ .../org/openrs2/cache/Js5CompressionTest.kt | 77 ++++++++++++++++++ .../cache/compression/empty-loc-bzip2.dat | Bin 0 -> 42 bytes .../compression/empty-loc-gzip-stored.dat | Bin 0 -> 33 bytes .../cache/compression/empty-loc-gzip.dat | Bin 0 -> 30 bytes .../cache/compression/empty-loc-lzma.dat | Bin 0 -> 20 bytes .../cache/compression/empty-loc-none.dat | Bin 0 -> 6 bytes 10 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 cache/src/test/resources/org/openrs2/cache/compression/empty-loc-bzip2.dat create mode 100644 cache/src/test/resources/org/openrs2/cache/compression/empty-loc-gzip-stored.dat create mode 100644 cache/src/test/resources/org/openrs2/cache/compression/empty-loc-gzip.dat create mode 100644 cache/src/test/resources/org/openrs2/cache/compression/empty-loc-lzma.dat create mode 100644 cache/src/test/resources/org/openrs2/cache/compression/empty-loc-none.dat diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index a23548f4..d2f2e4bd 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -43,6 +43,7 @@ public class CacheImporter @Inject constructor( public val encrypted: Boolean = uncompressed == null public val uncompressedLen: Int? = uncompressed?.readableBytes() public val uncompressedCrc32: Int? = uncompressed?.crc32() + public val emptyLoc: Boolean = Js5Compression.isEmptyLoc(compressed.slice()) public fun release() { compressed.release() @@ -696,7 +697,8 @@ public class CacheImporter @Inject constructor( uncompressed_length INTEGER NULL, uncompressed_crc32 INTEGER NULL, data BYTEA NOT NULL, - encrypted BOOLEAN NOT NULL + encrypted BOOLEAN NOT NULL, + empty_loc BOOLEAN NULL ) ON COMMIT DROP """.trimIndent() ).use { stmt -> @@ -719,8 +721,8 @@ public class CacheImporter @Inject constructor( connection.prepareStatement( """ - INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """.trimIndent() ).use { stmt -> for ((i, container) in containers.withIndex()) { @@ -731,6 +733,13 @@ public class CacheImporter @Inject constructor( stmt.setObject(5, container.uncompressedLen, Types.INTEGER) stmt.setObject(6, container.uncompressedCrc32, Types.INTEGER) stmt.setBoolean(7, container.encrypted) + + if (container.encrypted) { + stmt.setBoolean(8, container.emptyLoc) + } else { + stmt.setNull(8, Types.BOOLEAN) + } + stmt.addBatch() } @@ -739,8 +748,8 @@ public class CacheImporter @Inject constructor( connection.prepareStatement( """ - INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted) - SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted + INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc) + SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted, t.empty_loc FROM tmp_containers t LEFT JOIN containers c ON c.whirlpool = t.whirlpool WHERE c.whirlpool IS NULL diff --git a/archive/src/main/resources/org/openrs2/archive/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/V1__init.sql index 4954c868..a5b8eb58 100644 --- a/archive/src/main/resources/org/openrs2/archive/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/V1__init.sql @@ -34,6 +34,7 @@ CREATE TABLE containers ( uncompressed_length INTEGER NULL, uncompressed_crc32 INTEGER NULL, encrypted BOOLEAN NOT NULL, + empty_loc BOOLEAN NULL, key_id BIGINT NULL REFERENCES keys (id) ); @@ -168,7 +169,12 @@ GROUP BY a.master_index_id; CREATE UNIQUE INDEX ON master_index_archive_stats (master_index_id); CREATE MATERIALIZED VIEW master_index_group_stats (master_index_id, groups, valid_groups, keys, valid_keys) AS -SELECT v.master_index_id, COUNT(*), COUNT(c.id), COUNT(*) FILTER (WHERE c.encrypted), COUNT(k.id) +SELECT + v.master_index_id, + COUNT(*), + COUNT(c.id), + COUNT(*) FILTER (WHERE c.encrypted), + COUNT(*) FILTER (WHERE c.key_id IS NOT NULL OR (c.empty_loc IS NOT NULL AND c.empty_loc)) FROM master_index_valid_indexes v JOIN index_groups ig ON ig.container_id = v.container_id LEFT JOIN groups g ON g.archive_id = v.archive_id AND g.group_id = ig.group_id AND ( diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index 7f5ea87d..54a3df88 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -15,7 +15,7 @@ Name Indexes Groups - Keys + Keys[1] Download @@ -48,6 +48,17 @@ +

+ [1]: Map squares in the middle of the sea are unreachable by + normal players, making it impossible to obtain the keys for + their loc groups. However, the loc groups for these map squares + are empty. As XTEA does not hide the length of the compressed + data, this service infers which encrypted loc groups are empty + and includes them in the number of valid keys, regardless of + whether we know the key or not. After downloading a cache from + this service, a cache editor may be used to replace the empty + encrypted loc groups with unencrypted replacements. +

diff --git a/cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt b/cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt index 3331840e..712f7821 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt @@ -348,4 +348,42 @@ public object Js5Compression { return output.retain() } } + + public fun isEmptyLoc(buf: ByteBuf): Boolean { + val typeId = buf.readUnsignedByte().toInt() + val type = Js5CompressionType.fromOrdinal(typeId) + ?: throw IOException("Invalid compression type: $typeId") + + val len = buf.readInt() + if (len < 0) { + throw IOException("Length is negative: $len") + } + + if (type == Js5CompressionType.UNCOMPRESSED) { + if (buf.readableBytes() < len) { + throw IOException("Data truncated") + } + buf.skipBytes(len) + + // an empty loc group has a single byte + return len == 1 + } + + val lenWithUncompressedLen = len + 4 + if (buf.readableBytes() < lenWithUncompressedLen) { + throw IOException("Compressed data truncated") + } + buf.skipBytes(lenWithUncompressedLen) + + return when (type) { + Js5CompressionType.UNCOMPRESSED -> throw AssertionError() + Js5CompressionType.BZIP2 -> len == 33 + /* + * A single byte gzip compresses to 21 bytes with levels 1 to 9, + * and 24 bytes with level 0. + */ + Js5CompressionType.GZIP -> len == 21 || len == 24 + Js5CompressionType.LZMA -> len == 11 + } + } } diff --git a/cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt b/cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt index eec4cc54..f3de03d7 100644 --- a/cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt +++ b/cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt @@ -9,8 +9,10 @@ import java.io.IOException import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertFalse import kotlin.test.assertNotEquals import kotlin.test.assertNull +import kotlin.test.assertTrue class Js5CompressionTest { @Test @@ -605,6 +607,81 @@ class Js5CompressionTest { } } + @Test + fun testEmptyLocNone() { + read("empty-loc-none.dat").use { compressed -> + assertTrue(Js5Compression.isEmptyLoc(compressed)) + } + + read("none.dat").use { compressed -> + assertFalse(Js5Compression.isEmptyLoc(compressed)) + } + } + + @Test + fun testEmptyLocGzip() { + read("empty-loc-gzip.dat").use { compressed -> + assertTrue(Js5Compression.isEmptyLoc(compressed)) + } + + read("empty-loc-gzip-stored.dat").use { compressed -> + assertTrue(Js5Compression.isEmptyLoc(compressed)) + } + + read("gzip.dat").use { compressed -> + assertFalse(Js5Compression.isEmptyLoc(compressed)) + } + } + + @Test + fun testEmptyLocBzip2() { + read("empty-loc-bzip2.dat").use { compressed -> + assertTrue(Js5Compression.isEmptyLoc(compressed)) + } + + read("bzip2.dat").use { compressed -> + assertFalse(Js5Compression.isEmptyLoc(compressed)) + } + } + + @Test + fun testEmptyLocLzma() { + read("empty-loc-lzma.dat").use { compressed -> + assertTrue(Js5Compression.isEmptyLoc(compressed)) + } + + read("lzma.dat").use { compressed -> + assertFalse(Js5Compression.isEmptyLoc(compressed)) + } + } + + @Test + fun testEmptyLocInvalid() { + read("invalid-type.dat").use { compressed -> + assertFailsWith { + Js5Compression.isEmptyLoc(compressed) + } + } + + read("invalid-length.dat").use { compressed -> + assertFailsWith { + Js5Compression.isEmptyLoc(compressed) + } + } + + read("none-eof.dat").use { compressed -> + assertFailsWith { + Js5Compression.isEmptyLoc(compressed) + } + } + + read("compressed-underflow.dat").use { compressed -> + assertFailsWith { + Js5Compression.isEmptyLoc(compressed) + } + } + } + private fun read(name: String): ByteBuf { Js5CompressionTest::class.java.getResourceAsStream("compression/$name").use { input -> return Unpooled.wrappedBuffer(input.readAllBytes()) diff --git a/cache/src/test/resources/org/openrs2/cache/compression/empty-loc-bzip2.dat b/cache/src/test/resources/org/openrs2/cache/compression/empty-loc-bzip2.dat new file mode 100644 index 0000000000000000000000000000000000000000..2e2f6fdf2312a4effb0133546f6993d8606dc746 GIT binary patch literal 42 ycmZQ%U|>+}OgG=Z=*)2+W}{H45An~V?VSoTUwB_=mR_^wqU&DUN^bcQmJI+