Include empty loc groups in the total number of known keys

Signed-off-by: Graham <gpe@openrs2.org>
Graham 4 years ago
parent 2ac2ab8230
commit 416dabec4c
  1. 19
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
  2. 8
      archive/src/main/resources/org/openrs2/archive/V1__init.sql
  3. 13
      archive/src/main/resources/org/openrs2/archive/templates/caches/index.html
  4. 38
      cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt
  5. 77
      cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt
  6. BIN
      cache/src/test/resources/org/openrs2/cache/compression/empty-loc-bzip2.dat
  7. BIN
      cache/src/test/resources/org/openrs2/cache/compression/empty-loc-gzip-stored.dat
  8. BIN
      cache/src/test/resources/org/openrs2/cache/compression/empty-loc-gzip.dat
  9. BIN
      cache/src/test/resources/org/openrs2/cache/compression/empty-loc-lzma.dat
  10. BIN
      cache/src/test/resources/org/openrs2/cache/compression/empty-loc-none.dat

@ -43,6 +43,7 @@ public class CacheImporter @Inject constructor(
public val encrypted: Boolean = uncompressed == null public val encrypted: Boolean = uncompressed == null
public val uncompressedLen: Int? = uncompressed?.readableBytes() public val uncompressedLen: Int? = uncompressed?.readableBytes()
public val uncompressedCrc32: Int? = uncompressed?.crc32() public val uncompressedCrc32: Int? = uncompressed?.crc32()
public val emptyLoc: Boolean = Js5Compression.isEmptyLoc(compressed.slice())
public fun release() { public fun release() {
compressed.release() compressed.release()
@ -696,7 +697,8 @@ public class CacheImporter @Inject constructor(
uncompressed_length INTEGER NULL, uncompressed_length INTEGER NULL,
uncompressed_crc32 INTEGER NULL, uncompressed_crc32 INTEGER NULL,
data BYTEA NOT NULL, data BYTEA NOT NULL,
encrypted BOOLEAN NOT NULL encrypted BOOLEAN NOT NULL,
empty_loc BOOLEAN NULL
) ON COMMIT DROP ) ON COMMIT DROP
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
@ -719,8 +721,8 @@ public class CacheImporter @Inject constructor(
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted) INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
for ((i, container) in containers.withIndex()) { for ((i, container) in containers.withIndex()) {
@ -731,6 +733,13 @@ public class CacheImporter @Inject constructor(
stmt.setObject(5, container.uncompressedLen, Types.INTEGER) stmt.setObject(5, container.uncompressedLen, Types.INTEGER)
stmt.setObject(6, container.uncompressedCrc32, Types.INTEGER) stmt.setObject(6, container.uncompressedCrc32, Types.INTEGER)
stmt.setBoolean(7, container.encrypted) stmt.setBoolean(7, container.encrypted)
if (container.encrypted) {
stmt.setBoolean(8, container.emptyLoc)
} else {
stmt.setNull(8, Types.BOOLEAN)
}
stmt.addBatch() stmt.addBatch()
} }
@ -739,8 +748,8 @@ public class CacheImporter @Inject constructor(
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted) INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted, t.empty_loc
FROM tmp_containers t FROM tmp_containers t
LEFT JOIN containers c ON c.whirlpool = t.whirlpool LEFT JOIN containers c ON c.whirlpool = t.whirlpool
WHERE c.whirlpool IS NULL WHERE c.whirlpool IS NULL

@ -34,6 +34,7 @@ CREATE TABLE containers (
uncompressed_length INTEGER NULL, uncompressed_length INTEGER NULL,
uncompressed_crc32 INTEGER NULL, uncompressed_crc32 INTEGER NULL,
encrypted BOOLEAN NOT NULL, encrypted BOOLEAN NOT NULL,
empty_loc BOOLEAN NULL,
key_id BIGINT NULL REFERENCES keys (id) key_id BIGINT NULL REFERENCES keys (id)
); );
@ -168,7 +169,12 @@ GROUP BY a.master_index_id;
CREATE UNIQUE INDEX ON master_index_archive_stats (master_index_id); CREATE UNIQUE INDEX ON master_index_archive_stats (master_index_id);
CREATE MATERIALIZED VIEW master_index_group_stats (master_index_id, groups, valid_groups, keys, valid_keys) AS CREATE MATERIALIZED VIEW master_index_group_stats (master_index_id, groups, valid_groups, keys, valid_keys) AS
SELECT v.master_index_id, COUNT(*), COUNT(c.id), COUNT(*) FILTER (WHERE c.encrypted), COUNT(k.id) SELECT
v.master_index_id,
COUNT(*),
COUNT(c.id),
COUNT(*) FILTER (WHERE c.encrypted),
COUNT(*) FILTER (WHERE c.key_id IS NOT NULL OR (c.empty_loc IS NOT NULL AND c.empty_loc))
FROM master_index_valid_indexes v FROM master_index_valid_indexes v
JOIN index_groups ig ON ig.container_id = v.container_id JOIN index_groups ig ON ig.container_id = v.container_id
LEFT JOIN groups g ON g.archive_id = v.archive_id AND g.group_id = ig.group_id AND ( LEFT JOIN groups g ON g.archive_id = v.archive_id AND g.group_id = ig.group_id AND (

@ -15,7 +15,7 @@
<th>Name</th> <th>Name</th>
<th>Indexes</th> <th>Indexes</th>
<th>Groups</th> <th>Groups</th>
<th>Keys</th> <th>Keys<sup><a href="#empty-locs">[1]</a></sup></th>
<th>Download</th> <th>Download</th>
</tr> </tr>
</thead> </thead>
@ -48,6 +48,17 @@
</tr> </tr>
</tbody> </tbody>
</table> </table>
<p id="empty-locs">
[1]: Map squares in the middle of the sea are unreachable by
normal players, making it impossible to obtain the keys for
their loc groups. However, the loc groups for these map squares
are empty. As XTEA does not hide the length of the compressed
data, this service infers which encrypted loc groups are empty
and includes them in the number of valid keys, regardless of
whether we know the key or not. After downloading a cache from
this service, a cache editor may be used to replace the empty
encrypted loc groups with unencrypted replacements.
</p>
</main> </main>
</body> </body>
</html> </html>

@ -348,4 +348,42 @@ public object Js5Compression {
return output.retain() return output.retain()
} }
} }
public fun isEmptyLoc(buf: ByteBuf): Boolean {
val typeId = buf.readUnsignedByte().toInt()
val type = Js5CompressionType.fromOrdinal(typeId)
?: throw IOException("Invalid compression type: $typeId")
val len = buf.readInt()
if (len < 0) {
throw IOException("Length is negative: $len")
}
if (type == Js5CompressionType.UNCOMPRESSED) {
if (buf.readableBytes() < len) {
throw IOException("Data truncated")
}
buf.skipBytes(len)
// an empty loc group has a single byte
return len == 1
}
val lenWithUncompressedLen = len + 4
if (buf.readableBytes() < lenWithUncompressedLen) {
throw IOException("Compressed data truncated")
}
buf.skipBytes(lenWithUncompressedLen)
return when (type) {
Js5CompressionType.UNCOMPRESSED -> throw AssertionError()
Js5CompressionType.BZIP2 -> len == 33
/*
* A single byte gzip compresses to 21 bytes with levels 1 to 9,
* and 24 bytes with level 0.
*/
Js5CompressionType.GZIP -> len == 21 || len == 24
Js5CompressionType.LZMA -> len == 11
}
}
} }

@ -9,8 +9,10 @@ import java.io.IOException
import kotlin.test.Test import kotlin.test.Test
import kotlin.test.assertEquals import kotlin.test.assertEquals
import kotlin.test.assertFailsWith import kotlin.test.assertFailsWith
import kotlin.test.assertFalse
import kotlin.test.assertNotEquals import kotlin.test.assertNotEquals
import kotlin.test.assertNull import kotlin.test.assertNull
import kotlin.test.assertTrue
class Js5CompressionTest { class Js5CompressionTest {
@Test @Test
@ -605,6 +607,81 @@ class Js5CompressionTest {
} }
} }
@Test
fun testEmptyLocNone() {
read("empty-loc-none.dat").use { compressed ->
assertTrue(Js5Compression.isEmptyLoc(compressed))
}
read("none.dat").use { compressed ->
assertFalse(Js5Compression.isEmptyLoc(compressed))
}
}
@Test
fun testEmptyLocGzip() {
read("empty-loc-gzip.dat").use { compressed ->
assertTrue(Js5Compression.isEmptyLoc(compressed))
}
read("empty-loc-gzip-stored.dat").use { compressed ->
assertTrue(Js5Compression.isEmptyLoc(compressed))
}
read("gzip.dat").use { compressed ->
assertFalse(Js5Compression.isEmptyLoc(compressed))
}
}
@Test
fun testEmptyLocBzip2() {
read("empty-loc-bzip2.dat").use { compressed ->
assertTrue(Js5Compression.isEmptyLoc(compressed))
}
read("bzip2.dat").use { compressed ->
assertFalse(Js5Compression.isEmptyLoc(compressed))
}
}
@Test
fun testEmptyLocLzma() {
read("empty-loc-lzma.dat").use { compressed ->
assertTrue(Js5Compression.isEmptyLoc(compressed))
}
read("lzma.dat").use { compressed ->
assertFalse(Js5Compression.isEmptyLoc(compressed))
}
}
@Test
fun testEmptyLocInvalid() {
read("invalid-type.dat").use { compressed ->
assertFailsWith<IOException> {
Js5Compression.isEmptyLoc(compressed)
}
}
read("invalid-length.dat").use { compressed ->
assertFailsWith<IOException> {
Js5Compression.isEmptyLoc(compressed)
}
}
read("none-eof.dat").use { compressed ->
assertFailsWith<IOException> {
Js5Compression.isEmptyLoc(compressed)
}
}
read("compressed-underflow.dat").use { compressed ->
assertFailsWith<IOException> {
Js5Compression.isEmptyLoc(compressed)
}
}
}
private fun read(name: String): ByteBuf { private fun read(name: String): ByteBuf {
Js5CompressionTest::class.java.getResourceAsStream("compression/$name").use { input -> Js5CompressionTest::class.java.getResourceAsStream("compression/$name").use { input ->
return Unpooled.wrappedBuffer(input.readAllBytes()) return Unpooled.wrappedBuffer(input.readAllBytes())

Loading…
Cancel
Save