From 18d7fdc34305b5cc0e7731248be64b8d1440b887 Mon Sep 17 00:00:00 2001 From: Graham Date: Wed, 24 Mar 2021 17:03:55 +0000 Subject: [PATCH] Resolve collisions by prioritising groups from the same source This doesn't fix all cases, as a master index can have multiple sources, each with a distinct copy of the same (archive, group, checksum, version) tuple. However, it's probably as good as we'll be able to do automatically - and it'll work particularly well for master indexes downloaded directly over JS5, where we won't have done multiple imports of the same cache. Signed-off-by: Graham --- .../openrs2/archive/cache/CacheImporter.kt | 4 +- .../openrs2/archive/migrations/V1__init.sql | 41 ++++++++++++++++--- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt index 2f3e6514..1693e174 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt @@ -247,7 +247,7 @@ public class CacheImporter @Inject constructor( FROM master_index_archives a LEFT JOIN master_index_archives a2 ON a2.master_index_id = ? AND a2.archive_id = a.archive_id AND a2.crc32 = a.crc32 AND a2.version = a.version - LEFT JOIN LATERAL resolve_index(a2.archive_id, a2.crc32, a2.version) c ON TRUE + LEFT JOIN LATERAL resolve_index(a2.master_index_id, a2.archive_id, a2.crc32, a2.version) c ON TRUE WHERE a.master_index_id = ? ORDER BY a.archive_id ASC """.trimIndent() @@ -308,7 +308,7 @@ public class CacheImporter @Inject constructor( i.archive_id = ? LEFT JOIN index_groups ig2 ON ig2.container_id = i.container_id AND ig2.group_id = ig.group_id AND ig2.crc32 = ig.crc32 AND ig2.version = ig.version - LEFT JOIN LATERAL resolve_group(i.archive_id, ig2.group_id, ig2.crc32, ig2.version) c ON TRUE + LEFT JOIN LATERAL resolve_group(i.master_index_id, i.archive_id, ig2.group_id, ig2.crc32, ig2.version) c ON TRUE WHERE ig.container_id = ? AND c.id IS NULL ORDER BY ig.group_id ASC """.trimIndent() diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql index 41dffefb..4b3f3a86 100644 --- a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql @@ -163,11 +163,26 @@ CREATE TABLE names ( CREATE UNIQUE INDEX ON names (hash, name); -CREATE FUNCTION resolve_index(archive_id uint1, crc32 INTEGER, version INTEGER) RETURNS containers AS $$ +CREATE FUNCTION resolve_index(master_index_id INTEGER, archive_id uint1, crc32 INTEGER, version INTEGER) RETURNS containers AS $$ #variable_conflict use_variable DECLARE resolved containers%ROWTYPE; BEGIN + SELECT c.* + INTO resolved + FROM source_groups g + JOIN containers c ON c.id = g.container_id + JOIN indexes i ON i.container_id = c.id + JOIN sources s ON s.id = g.source_id + WHERE g.archive_id = 255 AND g.group_id = archive_id::INTEGER AND c.crc32 = crc32 AND g.version = version AND + NOT g.version_truncated AND i.version = version AND s.master_index_id = master_index_id + ORDER BY c.id ASC + LIMIT 1; + + IF FOUND THEN + RETURN resolved; + END IF; + SELECT c.* INTO resolved FROM groups g @@ -182,11 +197,25 @@ BEGIN END; $$ LANGUAGE plpgsql STABLE STRICT PARALLEL SAFE; -CREATE FUNCTION resolve_group(archive_id uint1, group_id INTEGER, crc32 INTEGER, version INTEGER) RETURNS containers AS $$ +CREATE FUNCTION resolve_group(master_index_id INTEGER, archive_id uint1, group_id INTEGER, crc32 INTEGER, version INTEGER) RETURNS containers AS $$ #variable_conflict use_variable DECLARE resolved containers%ROWTYPE; BEGIN + SELECT c.* + INTO resolved + FROM source_groups g + JOIN containers c ON c.id = g.container_id + JOIN sources s ON s.id = g.source_id + WHERE g.archive_id = archive_id AND g.group_id = group_id AND c.crc32 = crc32 AND g.version = version AND + NOT g.version_truncated AND s.master_index_id = master_index_id + ORDER BY c.id ASC + LIMIT 1; + + IF FOUND THEN + RETURN resolved; + END IF; + SELECT c.* INTO resolved FROM groups g @@ -217,7 +246,7 @@ CREATE VIEW resolved_indexes AS SELECT m.id AS master_index_id, a.archive_id, c.data, c.id AS container_id FROM master_indexes m JOIN master_index_archives a ON a.master_index_id = m.id -JOIN LATERAL resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE; +JOIN LATERAL resolve_index(m.id, a.archive_id, a.crc32, a.version) c ON TRUE; CREATE VIEW resolved_groups (master_index_id, archive_id, group_id, name_hash, version, data, key_id) AS WITH i AS NOT MATERIALIZED ( @@ -230,7 +259,7 @@ UNION ALL SELECT i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.key_id FROM i JOIN index_groups ig ON ig.container_id = i.container_id -JOIN LATERAL resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE; +JOIN LATERAL resolve_group(i.master_index_id, i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE; CREATE MATERIALIZED VIEW master_index_stats ( master_index_id, @@ -256,7 +285,7 @@ LEFT JOIN ( COUNT(*) FILTER (WHERE c.id IS NOT NULL OR (a.version = 0 AND a.crc32 = 0)) AS valid_indexes, COUNT(*) AS indexes FROM master_index_archives a - LEFT JOIN LATERAL resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE + LEFT JOIN LATERAL resolve_index(a.master_index_id, a.archive_id, a.crc32, a.version) c ON TRUE GROUP BY a.master_index_id ) a ON a.master_index_id = m.id LEFT JOIN ( @@ -268,7 +297,7 @@ LEFT JOIN ( COUNT(*) FILTER (WHERE c.encrypted) AS keys FROM resolved_indexes i JOIN index_groups ig ON ig.container_id = i.container_id - LEFT JOIN LATERAL resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE + LEFT JOIN LATERAL resolve_group(i.master_index_id, i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE LEFT JOIN keys k ON k.id = c.key_id GROUP BY i.master_index_id ) g ON g.master_index_id = m.id;