From dfb1f3c0e6bb0d489862a819c9748b6e4335325b Mon Sep 17 00:00:00 2001
From: Graham <gpe@openrs2.org>
Date: Tue, 25 Jul 2023 19:59:43 +0100
Subject: [PATCH] Skip containers that already exist during import

I'm hoping this will speed up importing large RS3 caches.

Signed-off-by: Graham <gpe@openrs2.org>
---
 .../openrs2/archive/cache/CacheImporter.kt    | 80 +++++++++++++++++--
 1 file changed, 73 insertions(+), 7 deletions(-)

diff --git a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
index 74b4ab3a..31d374aa 100644
--- a/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
+++ b/archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt
@@ -821,6 +821,17 @@ public class CacheImporter @Inject constructor(
             stmt.execute()
         }
 
+        connection.prepareStatement(
+            """
+            CREATE TEMPORARY TABLE tmp_container_hashes (
+                index INTEGER NOT NULL,
+                whirlpool BYTEA NOT NULL
+            ) ON COMMIT DROP
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.execute()
+        }
+
         connection.prepareStatement(
             """
             CREATE TEMPORARY TABLE tmp_containers (
@@ -859,12 +870,59 @@ public class CacheImporter @Inject constructor(
     private fun addContainers(connection: Connection, containers: List<Container>): List<Long> {
         connection.prepareStatement(
             """
-            TRUNCATE TABLE tmp_containers
+            TRUNCATE TABLE tmp_containers, tmp_container_hashes
             """.trimIndent()
         ).use { stmt ->
             stmt.execute()
         }
 
+        connection.prepareStatement(
+            """
+            INSERT INTO tmp_container_hashes (index, whirlpool)
+            VALUES (?, ?)
+        """.trimIndent()
+        ).use { stmt ->
+            for ((i, container) in containers.withIndex()) {
+                stmt.setInt(1, i)
+                stmt.setBytes(2, container.whirlpool)
+
+                stmt.addBatch()
+            }
+
+            stmt.executeBatch()
+        }
+
+        val ids = mutableListOf<Long?>()
+        var count = 0
+
+        connection.prepareStatement(
+            """
+            SELECT c.id
+            FROM tmp_container_hashes t
+            LEFT JOIN containers c ON c.whirlpool = t.whirlpool
+            ORDER BY t.index ASC
+        """.trimIndent()
+        ).use { stmt ->
+            stmt.executeQuery().use { rows ->
+                while (rows.next()) {
+                    val id = rows.getLong(1)
+                    if (rows.wasNull()) {
+                        ids += null
+                    } else {
+                        ids += id
+                        count++
+                    }
+                }
+            }
+        }
+
+        check(ids.size == containers.size)
+
+        if (count == containers.size) {
+            @Suppress("UNCHECKED_CAST")
+            return ids as List<Long>
+        }
+
         connection.prepareStatement(
             """
             INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
@@ -872,6 +930,10 @@ public class CacheImporter @Inject constructor(
             """.trimIndent()
         ).use { stmt ->
             for ((i, container) in containers.withIndex()) {
+                if (ids[i] != null) {
+                    continue
+                }
+
                 stmt.setInt(1, i)
                 stmt.setInt(2, container.crc32)
                 stmt.setBytes(3, container.whirlpool)
@@ -905,11 +967,9 @@ public class CacheImporter @Inject constructor(
             stmt.execute()
         }
 
-        val ids = mutableListOf<Long>()
-
         connection.prepareStatement(
             """
-            SELECT c.id
+            SELECT t.index, c.id
             FROM tmp_containers t
             JOIN containers c ON c.whirlpool = t.whirlpool
             ORDER BY t.index ASC
@@ -917,13 +977,19 @@ public class CacheImporter @Inject constructor(
         ).use { stmt ->
             stmt.executeQuery().use { rows ->
                 while (rows.next()) {
-                    ids += rows.getLong(1)
+                    val index = rows.getInt(1)
+                    val id = rows.getLong(2)
+
+                    ids[index] = id
+                    count++
                 }
             }
         }
 
-        check(ids.size == containers.size)
-        return ids
+        check(count == containers.size)
+
+        @Suppress("UNCHECKED_CAST")
+        return ids as List<Long>
     }
 
     private fun addBlob(connection: Connection, blob: Blob): Long {