Skip containers that already exist during import

I'm hoping this will speed up importing large RS3 caches.

Signed-off-by: Graham <gpe@openrs2.org>
Graham 1 year ago
parent 28a9667471
commit dfb1f3c0e6
  1. 80
      archive/src/main/kotlin/org/openrs2/archive/cache/CacheImporter.kt

@ -821,6 +821,17 @@ public class CacheImporter @Inject constructor(
stmt.execute() stmt.execute()
} }
connection.prepareStatement(
"""
CREATE TEMPORARY TABLE tmp_container_hashes (
index INTEGER NOT NULL,
whirlpool BYTEA NOT NULL
) ON COMMIT DROP
""".trimIndent()
).use { stmt ->
stmt.execute()
}
connection.prepareStatement( connection.prepareStatement(
""" """
CREATE TEMPORARY TABLE tmp_containers ( CREATE TEMPORARY TABLE tmp_containers (
@ -859,12 +870,59 @@ public class CacheImporter @Inject constructor(
private fun addContainers(connection: Connection, containers: List<Container>): List<Long> { private fun addContainers(connection: Connection, containers: List<Container>): List<Long> {
connection.prepareStatement( connection.prepareStatement(
""" """
TRUNCATE TABLE tmp_containers TRUNCATE TABLE tmp_containers, tmp_container_hashes
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
stmt.execute() stmt.execute()
} }
connection.prepareStatement(
"""
INSERT INTO tmp_container_hashes (index, whirlpool)
VALUES (?, ?)
""".trimIndent()
).use { stmt ->
for ((i, container) in containers.withIndex()) {
stmt.setInt(1, i)
stmt.setBytes(2, container.whirlpool)
stmt.addBatch()
}
stmt.executeBatch()
}
val ids = mutableListOf<Long?>()
var count = 0
connection.prepareStatement(
"""
SELECT c.id
FROM tmp_container_hashes t
LEFT JOIN containers c ON c.whirlpool = t.whirlpool
ORDER BY t.index ASC
""".trimIndent()
).use { stmt ->
stmt.executeQuery().use { rows ->
while (rows.next()) {
val id = rows.getLong(1)
if (rows.wasNull()) {
ids += null
} else {
ids += id
count++
}
}
}
}
check(ids.size == containers.size)
if (count == containers.size) {
@Suppress("UNCHECKED_CAST")
return ids as List<Long>
}
connection.prepareStatement( connection.prepareStatement(
""" """
INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc) INSERT INTO tmp_containers (index, crc32, whirlpool, data, uncompressed_length, uncompressed_crc32, encrypted, empty_loc)
@ -872,6 +930,10 @@ public class CacheImporter @Inject constructor(
""".trimIndent() """.trimIndent()
).use { stmt -> ).use { stmt ->
for ((i, container) in containers.withIndex()) { for ((i, container) in containers.withIndex()) {
if (ids[i] != null) {
continue
}
stmt.setInt(1, i) stmt.setInt(1, i)
stmt.setInt(2, container.crc32) stmt.setInt(2, container.crc32)
stmt.setBytes(3, container.whirlpool) stmt.setBytes(3, container.whirlpool)
@ -905,11 +967,9 @@ public class CacheImporter @Inject constructor(
stmt.execute() stmt.execute()
} }
val ids = mutableListOf<Long>()
connection.prepareStatement( connection.prepareStatement(
""" """
SELECT c.id SELECT t.index, c.id
FROM tmp_containers t FROM tmp_containers t
JOIN containers c ON c.whirlpool = t.whirlpool JOIN containers c ON c.whirlpool = t.whirlpool
ORDER BY t.index ASC ORDER BY t.index ASC
@ -917,13 +977,19 @@ public class CacheImporter @Inject constructor(
).use { stmt -> ).use { stmt ->
stmt.executeQuery().use { rows -> stmt.executeQuery().use { rows ->
while (rows.next()) { while (rows.next()) {
ids += rows.getLong(1) val index = rows.getInt(1)
val id = rows.getLong(2)
ids[index] = id
count++
} }
} }
} }
check(ids.size == containers.size) check(count == containers.size)
return ids
@Suppress("UNCHECKED_CAST")
return ids as List<Long>
} }
private fun addBlob(connection: Connection, blob: Blob): Long { private fun addBlob(connection: Connection, blob: Blob): Long {

Loading…
Cancel
Save