Include empty loc groups in the total number of known keys

Signed-off-by: Graham <gpe@openrs2.org>

archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt

@@ -43,6 +43,7 @@ public class CacheImporter @Inject constructor(
         public val encrypted: Boolean = uncompressed == null
         public val uncompressedLen: Int? = uncompressed?.readableBytes()
         public val uncompressedCrc32: Int? = uncompressed?.crc32()
+        public val emptyLoc: Boolean = Js5Compression.isEmptyLoc(compressed.slice())
 
         public fun release() {
             compressed.release()
@@ -696,7 +697,8 @@ public class CacheImporter @Inject constructor(
                 uncompressed_length INTEGER NULL,
                 uncompressed_crc32 INTEGER NULL,
                 data BYTEA NOT NULL,
-                encrypted BOOLEAN NOT NULL
+                encrypted BOOLEAN NOT NULL,
+                empty_loc BOOLEAN NULL
             ) ON COMMIT DROP
         """.trimIndent()
         ).use { stmt ->
@@ -719,8 +721,8 @@ public class CacheImporter @Inject constructor(
 
         connection.prepareStatement(
             """
-            INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted)
-            VALUES (?, ?, ?, ?, ?, ?, ?)
+            INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
         """.trimIndent()
         ).use { stmt ->
             for ((i, container) in containers.withIndex()) {
@@ -731,6 +733,13 @@ public class CacheImporter @Inject constructor(
                 stmt.setObject(5, container.uncompressedLen, Types.INTEGER)
                 stmt.setObject(6, container.uncompressedCrc32, Types.INTEGER)
                 stmt.setBoolean(7, container.encrypted)
+
+                if (container.encrypted) {
+                    stmt.setBoolean(8, container.emptyLoc)
+                } else {
+                    stmt.setNull(8, Types.BOOLEAN)
+                }
+
                 stmt.addBatch()
             }
 
@@ -739,8 +748,8 @@ public class CacheImporter @Inject constructor(
 
         connection.prepareStatement(
             """
-            INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted)
-            SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted
+            INSERT INTO containers (crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
+            SELECT t.crc32, t.whirlpool, t.data, t.uncompressed_length, t.uncompressed_crc32, t.encrypted, t.empty_loc
             FROM tmp_containers t
             LEFT JOIN containers c ON c.whirlpool = t.whirlpool
             WHERE c.whirlpool IS NULL

archive/src/main/resources/org/openrs2/archive/V1__init.sql

@@ -34,6 +34,7 @@ CREATE TABLE containers (
     uncompressed_length INTEGER NULL,
     uncompressed_crc32 INTEGER NULL,
     encrypted BOOLEAN NOT NULL,
+    empty_loc BOOLEAN NULL,
     key_id BIGINT NULL REFERENCES keys (id)
 );
 
@@ -168,7 +169,12 @@ GROUP BY a.master_index_id;
 CREATE UNIQUE INDEX ON master_index_archive_stats (master_index_id);
 
 CREATE MATERIALIZED VIEW master_index_group_stats (master_index_id, groups, valid_groups, keys, valid_keys) AS
-SELECT v.master_index_id, COUNT(*), COUNT(c.id), COUNT(*) FILTER (WHERE c.encrypted), COUNT(k.id)
+SELECT
+   v.master_index_id,
+   COUNT(*),
+   COUNT(c.id),
+   COUNT(*) FILTER (WHERE c.encrypted),
+   COUNT(*) FILTER (WHERE c.key_id IS NOT NULL OR (c.empty_loc IS NOT NULL AND c.empty_loc))
 FROM master_index_valid_indexes v
 JOIN index_groups ig ON ig.container_id = v.container_id
 LEFT JOIN groups g ON g.archive_id = v.archive_id AND g.group_id = ig.group_id AND (

archive/src/main/resources/org/openrs2/archive/templates/caches/index.html

@@ -15,7 +15,7 @@
 						<th>Name</th>
 						<th>Indexes</th>
 						<th>Groups</th>
-						<th>Keys</th>
+						<th>Keys<sup><a href="#empty-locs">[1]</a></sup></th>
 						<th>Download</th>
 					</tr>
 				</thead>
@@ -48,6 +48,17 @@
 					</tr>
 				</tbody>
 			</table>
+			<p id="empty-locs">
+				[1]: Map squares in the middle of the sea are unreachable by
+				normal players, making it impossible to obtain the keys for
+				their loc groups. However, the loc groups for these map squares
+				are empty. As XTEA does not hide the length of the compressed
+				data, this service infers which encrypted loc groups are empty
+				and includes them in the number of valid keys, regardless of
+				whether we know the key or not. After downloading a cache from
+				this service, a cache editor may be used to replace the empty
+				encrypted loc groups with unencrypted replacements.
+			</p>
 		</main>
 	</body>
 </html>

cache/src/main/kotlin/org/openrs2/cache/Js5Compression.kt

@@ -348,4 +348,42 @@ public object Js5Compression {
             return output.retain()
         }
     }
+
+    public fun isEmptyLoc(buf: ByteBuf): Boolean {
+        val typeId = buf.readUnsignedByte().toInt()
+        val type = Js5CompressionType.fromOrdinal(typeId)
+            ?: throw IOException("Invalid compression type: $typeId")
+
+        val len = buf.readInt()
+        if (len < 0) {
+            throw IOException("Length is negative: $len")
+        }
+
+        if (type == Js5CompressionType.UNCOMPRESSED) {
+            if (buf.readableBytes() < len) {
+                throw IOException("Data truncated")
+            }
+            buf.skipBytes(len)
+
+            // an empty loc group has a single byte
+            return len == 1
+        }
+
+        val lenWithUncompressedLen = len + 4
+        if (buf.readableBytes() < lenWithUncompressedLen) {
+            throw IOException("Compressed data truncated")
+        }
+        buf.skipBytes(lenWithUncompressedLen)
+
+        return when (type) {
+            Js5CompressionType.UNCOMPRESSED -> throw AssertionError()
+            Js5CompressionType.BZIP2 -> len == 33
+            /*
+             * A single byte gzip compresses to 21 bytes with levels 1 to 9,
+             * and 24 bytes with level 0.
+             */
+            Js5CompressionType.GZIP -> len == 21 || len == 24
+            Js5CompressionType.LZMA -> len == 11
+        }
+    }
 }

cache/src/test/kotlin/org/openrs2/cache/Js5CompressionTest.kt

@@ -9,8 +9,10 @@ import java.io.IOException
 import kotlin.test.Test
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
+import kotlin.test.assertFalse
 import kotlin.test.assertNotEquals
 import kotlin.test.assertNull
+import kotlin.test.assertTrue
 
 class Js5CompressionTest {
     @Test
@@ -605,6 +607,81 @@ class Js5CompressionTest {
         }
     }
 
+    @Test
+    fun testEmptyLocNone() {
+        read("empty-loc-none.dat").use { compressed ->
+            assertTrue(Js5Compression.isEmptyLoc(compressed))
+        }
+
+        read("none.dat").use { compressed ->
+            assertFalse(Js5Compression.isEmptyLoc(compressed))
+        }
+    }
+
+    @Test
+    fun testEmptyLocGzip() {
+        read("empty-loc-gzip.dat").use { compressed ->
+            assertTrue(Js5Compression.isEmptyLoc(compressed))
+        }
+
+        read("empty-loc-gzip-stored.dat").use { compressed ->
+            assertTrue(Js5Compression.isEmptyLoc(compressed))
+        }
+
+        read("gzip.dat").use { compressed ->
+            assertFalse(Js5Compression.isEmptyLoc(compressed))
+        }
+    }
+
+    @Test
+    fun testEmptyLocBzip2() {
+        read("empty-loc-bzip2.dat").use { compressed ->
+            assertTrue(Js5Compression.isEmptyLoc(compressed))
+        }
+
+        read("bzip2.dat").use { compressed ->
+            assertFalse(Js5Compression.isEmptyLoc(compressed))
+        }
+    }
+
+    @Test
+    fun testEmptyLocLzma() {
+        read("empty-loc-lzma.dat").use { compressed ->
+            assertTrue(Js5Compression.isEmptyLoc(compressed))
+        }
+
+        read("lzma.dat").use { compressed ->
+            assertFalse(Js5Compression.isEmptyLoc(compressed))
+        }
+    }
+
+    @Test
+    fun testEmptyLocInvalid() {
+        read("invalid-type.dat").use { compressed ->
+            assertFailsWith<IOException> {
+                Js5Compression.isEmptyLoc(compressed)
+            }
+        }
+
+        read("invalid-length.dat").use { compressed ->
+            assertFailsWith<IOException> {
+                Js5Compression.isEmptyLoc(compressed)
+            }
+        }
+
+        read("none-eof.dat").use { compressed ->
+            assertFailsWith<IOException> {
+                Js5Compression.isEmptyLoc(compressed)
+            }
+        }
+
+        read("compressed-underflow.dat").use { compressed ->
+            assertFailsWith<IOException> {
+                Js5Compression.isEmptyLoc(compressed)
+            }
+        }
+    }
+
     private fun read(name: String): ByteBuf {
         Js5CompressionTest::class.java.getResourceAsStream("compression/$name").use { input ->
             return Unpooled.wrappedBuffer(input.readAllBytes())