From cbb2a903886e7521ff23d76dbfea2f6e36c2c2e4 Mon Sep 17 00:00:00 2001 From: Graham Date: Tue, 24 Aug 2021 12:21:34 +0100 Subject: [PATCH] Disable .dat2/.idx link if the cache is too big for a DiskStore Signed-off-by: Graham --- .../openrs2/archive/cache/CacheExporter.kt | 24 +++++-- .../openrs2/archive/migrations/V6__blocks.sql | 67 +++++++++++++++++++ .../archive/templates/caches/index.html | 5 +- .../archive/templates/caches/show.html | 3 +- .../kotlin/org/openrs2/cache/DiskStore.kt | 2 +- 5 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 archive/src/main/resources/org/openrs2/archive/migrations/V6__blocks.sql diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt index cd16f7ff..a4c24d6a 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheExporter.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonProperty import io.netty.buffer.ByteBufAllocator import io.netty.buffer.Unpooled import org.openrs2.buffer.use +import org.openrs2.cache.DiskStore import org.openrs2.cache.Js5Archive import org.openrs2.cache.Js5Compression import org.openrs2.cache.Js5MasterIndex @@ -29,7 +30,8 @@ public class CacheExporter @Inject constructor( val groups: Long, val validKeys: Long, val keys: Long, - val size: Long + val size: Long, + val blocks: Long ) { public val allIndexesValid: Boolean = indexes == validIndexes && indexes != 0L public val validIndexesFraction: Double = if (indexes == 0L) { @@ -51,6 +53,12 @@ public class CacheExporter @Inject constructor( } else { validKeys.toDouble() / keys } + + /* + * The max block ID is conveniently also the max number of blocks, as + * zero is reserved. + */ + public val diskStoreValid: Boolean = blocks <= DiskStore.MAX_BLOCK } public data class Build(val major: Int, val minor: Int?) : Comparable { @@ -139,13 +147,14 @@ public class CacheExporter @Inject constructor( ms.groups, ms.valid_keys, ms.keys, - ms.size + ms.size, + ms.blocks FROM master_indexes m JOIN sources s ON s.master_index_id = m.id JOIN games g ON g.id = s.game_id LEFT JOIN master_index_stats ms ON ms.master_index_id = m.id GROUP BY m.id, g.name, ms.valid_indexes, ms.indexes, ms.valid_groups, ms.groups, ms.valid_keys, ms.keys, - ms.size + ms.size, ms.blocks ) t ORDER BY t.name ASC, t.builds[1] ASC, t.timestamp ASC """.trimIndent() @@ -168,7 +177,8 @@ public class CacheExporter @Inject constructor( val validKeys = rows.getLong(10) val keys = rows.getLong(11) val size = rows.getLong(12) - Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size) + val blocks = rows.getLong(13) + Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks) } else { null } @@ -205,7 +215,8 @@ public class CacheExporter @Inject constructor( ms.groups, ms.valid_keys, ms.keys, - ms.size + ms.size, + ms.blocks FROM master_indexes m JOIN containers c ON c.id = m.container_id LEFT JOIN master_index_stats ms ON ms.master_index_id = m.id @@ -237,7 +248,8 @@ public class CacheExporter @Inject constructor( val validKeys = rows.getLong(7) val keys = rows.getLong(8) val size = rows.getLong(9) - Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size) + val blocks = rows.getLong(10) + Stats(validIndexes, indexes, validGroups, groups, validKeys, keys, size, blocks) } } } diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V6__blocks.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V6__blocks.sql new file mode 100644 index 00000000..11925d14 --- /dev/null +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V6__blocks.sql @@ -0,0 +1,67 @@ +-- @formatter:off +CREATE FUNCTION group_blocks(group_id INTEGER, len INTEGER) RETURNS INTEGER AS $$ + SELECT CASE + WHEN len = 0 THEN 1 + WHEN group_id >= 65536 THEN (len + 509) / 510 + ELSE (len + 511) / 512 + END; +$$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; + +CREATE MATERIALIZED VIEW master_index_stats_new ( + master_index_id, + valid_indexes, + indexes, + valid_groups, + groups, + valid_keys, + keys, + size, + blocks +) AS +SELECT + m.id, + COALESCE(a.valid_indexes, 0), + COALESCE(a.indexes, 0), + COALESCE(g.valid_groups, 0), + COALESCE(g.groups, 0), + COALESCE(g.valid_keys, 0), + COALESCE(g.keys, 0), + COALESCE(a.size, 0) + COALESCE(g.size, 0), + COALESCE(a.blocks, 0) + COALESCE(g.blocks, 0) +FROM master_indexes m +LEFT JOIN ( + SELECT + a.master_index_id, + COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, + COUNT(*) AS indexes, + SUM(length(c.data)) FILTER (WHERE c.id IS NOT NULL) AS size, + SUM(group_blocks(a.archive_id, length(c.data))) FILTER (WHERE c.id IS NOT NULL) AS blocks + FROM master_index_archives a + LEFT JOIN resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE + GROUP BY a.master_index_id +) a ON a.master_index_id = m.id +LEFT JOIN ( + SELECT + i.master_index_id, + COUNT(*) FILTER (WHERE c.id IS NOT NULL) AS valid_groups, + COUNT(*) AS groups, + COUNT(*) FILTER (WHERE c.encrypted AND (c.key_id IS NOT NULL OR c.empty_loc)) AS valid_keys, + COUNT(*) FILTER (WHERE c.encrypted) AS keys, + SUM(length(c.data)) FILTER (WHERE c.id IS NOT NULL) AS size, + SUM(group_blocks(ig.group_id, length(c.data))) FILTER (WHERE c.id IS NOT NULL) AS blocks + FROM resolved_indexes i + JOIN index_groups ig ON ig.container_id = i.container_id + LEFT JOIN resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE + LEFT JOIN keys k ON k.id = c.key_id + GROUP BY i.master_index_id +) g ON g.master_index_id = m.id; + +CREATE UNIQUE INDEX ON master_index_stats_new (master_index_id); + +ALTER MATERIALIZED VIEW master_index_stats RENAME TO master_index_stats_old; +ALTER INDEX master_index_stats_master_index_id_idx RENAME TO master_index_stats_old_master_index_id_idx; + +ALTER MATERIALIZED VIEW master_index_stats_new RENAME TO master_index_stats; +ALTER INDEX master_index_stats_new_master_index_id_idx RENAME TO master_index_stats_master_index_id_idx; + +DROP MATERIALIZED VIEW master_index_stats_old; diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index c6a21299..4208940c 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -79,8 +79,9 @@ Download
    -
  • Cache (.dat2/.idx)
    • +
  • Cache (.dat2/.idx)
  • Cache (Flat file)
  • diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html index 2a616469..e3eba509 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html @@ -51,7 +51,8 @@
    - Cache (.dat2/.idx) Cache (Flat file) diff --git a/cache/src/main/kotlin/org/openrs2/cache/DiskStore.kt b/cache/src/main/kotlin/org/openrs2/cache/DiskStore.kt index e812c771..f5963b5c 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/DiskStore.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/DiskStore.kt @@ -472,7 +472,7 @@ public class DiskStore private constructor( internal const val EXTENDED_BLOCK_HEADER_SIZE = 10 internal const val EXTENDED_BLOCK_DATA_SIZE = 510 - internal const val MAX_BLOCK = (1 shl 24) - 1 + public const val MAX_BLOCK: Int = (1 shl 24) - 1 private val TEMP_BUFFER_SIZE = max(INDEX_ENTRY_SIZE, max(BLOCK_HEADER_SIZE, EXTENDED_BLOCK_HEADER_SIZE)) private const val INDEX_BUFFER_SIZE = INDEX_ENTRY_SIZE * 1000