From 93ee863e2004dddcb038e24d09033e6c926d9c53 Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 1 Apr 2021 22:29:45 +0100 Subject: [PATCH] Add initial support for plotting a map of valid/invalid XTEA keys Signed-off-by: Graham --- .../kotlin/org/openrs2/archive/map/Colors.kt | 107 +++++ .../kotlin/org/openrs2/archive/map/FloType.kt | 57 +++ .../kotlin/org/openrs2/archive/map/FluType.kt | 34 ++ .../org/openrs2/archive/map/MapRenderer.kt | 426 ++++++++++++++++++ .../openrs2/archive/web/CachesController.kt | 28 ++ .../org/openrs2/archive/web/WebServer.kt | 1 + .../openrs2/archive/migrations/V1__init.sql | 6 +- .../archive/templates/caches/index.html | 2 + .../archive/templates/caches/show.html | 2 + 9 files changed, 660 insertions(+), 3 deletions(-) create mode 100644 archive/src/main/kotlin/org/openrs2/archive/map/Colors.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/map/FloType.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/map/FluType.kt create mode 100644 archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt diff --git a/archive/src/main/kotlin/org/openrs2/archive/map/Colors.kt b/archive/src/main/kotlin/org/openrs2/archive/map/Colors.kt new file mode 100644 index 00000000..ab390124 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/map/Colors.kt @@ -0,0 +1,107 @@ +package org.openrs2.archive.map + +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow + +public object Colors { + private val HSL_TO_RGB = IntArray(65536) + private const val BRIGHTNESS = 0.8 + + init { + var i = 0 + for (h in 0 until 64) { + for (s in 0 until 8) { + for (l in 0 until 128) { + val hue = h.toDouble() / 64 + 0.0078125 + val saturation = s.toDouble() / 8 + 0.0625 + val lightness = l.toDouble() / 128 + HSL_TO_RGB[i++] = hslToRgb(hue, saturation, lightness) + } + } + } + } + + private fun hslToRgb(h: Double, s: Double, l: Double): Int { + var r = l + var g = l + var b = l + + if (s != 0.0) { + val q = if (l * 2 < 1) { + l * (s + 1) + } else { + l + s - (l * s) + } + + val p = l * 2 - q + + var tr = h + (1.0 / 3) + if (tr > 1) { + tr-- + } + + var tb = h - (1.0 / 3) + if (tb < 0) { + tb++ + } + + r = if (tr * 6 < 1) { + tr * (q - p) * 6 + p + } else if (tr * 2 < 1) { + q + } else if (tr * 3 < 2) { + (2.0 / 3 - tr) * (q - p) * 6 + p + } else { + p + } + + g = if (h * 6 < 1) { + h * (q - p) * 6 + p + } else if (h * 2 < 1) { + q + } else if (h * 3 < 2) { + (2.0 / 3 - h) * (q - p) * 6 + p + } else { + p + } + + b = if (tb * 6 < 1) { + tb * (q - p) * 6 + p + } else if (tb * 2 < 1) { + q + } else if (tb * 3 < 2) { + (2.0 / 3 - tb) * (q - p) * 6 + p + } else { + p + } + } + + val red = (r.pow(BRIGHTNESS) * 256).toInt() + val green = (g.pow(BRIGHTNESS) * 256).toInt() + val blue = (b.pow(BRIGHTNESS) * 256).toInt() + + var rgb = (red shl 16) or (green shl 8) or blue + if (rgb == 0) { + rgb = 1 + } + + return rgb + } + + public fun hslToRgb(hsl: Int): Int { + return HSL_TO_RGB[hsl] + } + + public fun multiplyLightness(hsl: Int, factor: Int): Int { + return when (hsl) { + -2 -> 12345678 + -1 -> 127 - min(max(factor, 0), 127) + else -> { + var l = ((hsl and 0x7F) * factor) shr 7 + l = min(max(l, 2), 126) + (hsl and 0xFF80) or l + } + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/map/FloType.kt b/archive/src/main/kotlin/org/openrs2/archive/map/FloType.kt new file mode 100644 index 00000000..d88b18b7 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/map/FloType.kt @@ -0,0 +1,57 @@ +package org.openrs2.archive.map + +import io.netty.buffer.ByteBuf + +public data class FloType( + var color: Int = 0, + var texture: Int = -1, + var blendColor: Int = -1 +) { + public companion object { + public fun read(buf: ByteBuf): FloType { + val type = FloType() + + while (true) { + val code = buf.readUnsignedByte().toInt() + if (code == 0) { + break + } else if (code == 1) { + type.color = buf.readUnsignedMedium() + } else if (code == 2) { + type.texture = buf.readUnsignedByte().toInt() + } else if (code == 3) { + type.texture = buf.readUnsignedShort() + if (type.texture == 65535) { + type.texture = -1 + } + } else if (code == 5) { + // empty + } else if (code == 7) { + type.blendColor = buf.readUnsignedMedium() + } else if (code == 8) { + // empty + } else if (code == 9) { + buf.skipBytes(2) + } else if (code == 10) { + // empty + } else if (code == 11) { + buf.skipBytes(1) + } else if (code == 12) { + // empty + } else if (code == 13) { + buf.skipBytes(3) + } else if (code == 14) { + buf.skipBytes(1) + } else if (code == 15) { + buf.skipBytes(2) + } else if (code == 16) { + buf.skipBytes(1) + } else { + throw IllegalArgumentException("Unsupported code: $code") + } + } + + return type + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/map/FluType.kt b/archive/src/main/kotlin/org/openrs2/archive/map/FluType.kt new file mode 100644 index 00000000..772f5977 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/map/FluType.kt @@ -0,0 +1,34 @@ +package org.openrs2.archive.map + +import io.netty.buffer.ByteBuf + +public data class FluType( + var color: Int = 0 +) { + public companion object { + public fun read(buf: ByteBuf): FluType { + val type = FluType() + + while (true) { + val code = buf.readUnsignedByte().toInt() + if (code == 0) { + break + } else if (code == 1) { + type.color = buf.readUnsignedMedium() + } else if (code == 2) { + buf.skipBytes(2) + } else if (code == 3) { + buf.skipBytes(2) + } else if (code == 4) { + // empty + } else if (code == 5) { + // empty + } else { + throw IllegalArgumentException("Unsupported code: $code") + } + } + + return type + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt b/archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt new file mode 100644 index 00000000..d8ae2418 --- /dev/null +++ b/archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt @@ -0,0 +1,426 @@ +package org.openrs2.archive.map + +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap +import org.openrs2.buffer.use +import org.openrs2.cache.Group +import org.openrs2.cache.Js5Archive +import org.openrs2.cache.Js5Compression +import org.openrs2.cache.Js5ConfigGroup +import org.openrs2.cache.Js5Index +import org.openrs2.db.Database +import java.awt.Color +import java.awt.Graphics2D +import java.awt.image.BufferedImage +import java.sql.Connection +import javax.inject.Inject +import kotlin.math.max +import kotlin.math.min + +public class MapRenderer @Inject constructor( + private val database: Database +) { + private enum class MapSquareState( + val outlineColor: Color + ) { + UNKNOWN(Color.RED), + EMPTY(Color.GRAY), + VALID(Color.GREEN); + + val fillColor = Color(outlineColor.red, outlineColor.green, outlineColor.blue, 128) + } + + public suspend fun render(masterIndexId: Int): BufferedImage { + return database.execute { connection -> + // read config index + val configIndex = readIndex(connection, masterIndexId, Js5Archive.CONFIG) + ?: throw IllegalArgumentException("Config index missing") + + // read FluType group + val underlayColors = mutableMapOf() + + val underlayGroup = configIndex[Js5ConfigGroup.FLUTYPE] + ?: throw IllegalArgumentException("FluType group missing in index") + + val underlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, underlayGroup) + ?: throw IllegalArgumentException("FluType group missing") + try { + for ((id, file) in underlayFiles) { + underlayColors[id] = FluType.read(file).color + } + } finally { + underlayFiles.values.forEach(ByteBuf::release) + } + + // read FloType group + val overlays = mutableMapOf() + + val overlayGroup = configIndex[Js5ConfigGroup.FLOTYPE] + ?: throw IllegalArgumentException("FloType group missing in index") + + val overlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, overlayGroup) + ?: throw IllegalArgumentException("FloType group missing") + try { + for ((id, file) in overlayFiles) { + overlays[id] = FloType.read(file) + } + } finally { + overlayFiles.values.forEach(ByteBuf::release) + } + + // read textures + val textures = mutableMapOf() + val materialsIndex = readIndex(connection, masterIndexId, Js5Archive.MATERIALS) + + if (materialsIndex != null) { + val materialsGroup = materialsIndex[0] + ?: throw IllegalArgumentException("Materials group missing in index") + + val materialsFiles = readGroup(connection, masterIndexId, Js5Archive.MATERIALS, materialsGroup) + ?: throw IllegalArgumentException("Materials group missing") + try { + val metadata = materialsFiles[0] + val len = metadata.readUnsignedShort() + + val ids = mutableSetOf() + for (id in 0 until len) { + if (metadata.readBoolean()) { + ids += id + } + } + + val use = mutableSetOf() + for (id in ids) { + if (metadata.readBoolean()) { + use += id + } + } + + metadata.skipBytes(ids.size * 7) + + for (id in ids) { + textures[id] = metadata.readUnsignedShort() + + if (id !in use) { + textures.remove(id) + } + } + } finally { + materialsFiles.values.forEach(ByteBuf::release) + } + } else { + val textureIndex = readIndex(connection, masterIndexId, Js5Archive.TEXTURES) + ?: throw IllegalArgumentException("Textures index missing") + + val textureGroup = textureIndex[0] + ?: throw IllegalArgumentException("Textures group missing from index") + + val textureFiles = readGroup(connection, masterIndexId, Js5Archive.TEXTURES, textureGroup) + ?: throw IllegalArgumentException("Textures group missing") + try { + for ((id, file) in textureFiles) { + textures[id] = file.readUnsignedShort() + } + } finally { + textureFiles.values.forEach(ByteBuf::release) + } + } + + // create overlay colors + val overlayColors = createOverlayColors(overlays, textures) + + // read loc encrypted/empty loc flags and keys and determine bounds of the map + var x0 = Int.MAX_VALUE + var x1 = Int.MIN_VALUE + var z0 = Int.MAX_VALUE + var z1 = Int.MIN_VALUE + val states = mutableMapOf() + + connection.prepareStatement( + """ + SELECT n.name, g.encrypted, g.empty_loc, g.key_id + FROM resolved_groups g + JOIN names n ON n.hash = g.name_hash + WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND + n.name ~ '^[lm](?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' + """.trimIndent() + ).use { stmt -> + stmt.setInt(1, masterIndexId) + + stmt.executeQuery().use { rows -> + while (rows.next()) { + val name = rows.getString(1) + val encrypted = rows.getBoolean(2) + val empty = rows.getBoolean(3) + var keyId: Long? = rows.getLong(4) + if (rows.wasNull()) { + keyId = null + } + + val (x, z) = getMapCoordinates(name) + x0 = min(x0, x) + x1 = max(x1, x) + z0 = min(z0, z) + z1 = max(z1, z) + + if (name.startsWith('l')) { + val mapSquare = getMapSquare(x, z) + states[mapSquare] = if (!encrypted || keyId != null) { + MapSquareState.VALID + } else if (empty) { + MapSquareState.EMPTY + } else { + MapSquareState.UNKNOWN + } + } + } + } + } + + if (x0 == Int.MAX_VALUE) { + throw IllegalArgumentException("Map empty") + } + + // read and render maps + val image = BufferedImage( + ((x1 - x0) + 1) * MAP_SIZE, + ((z1 - z0) + 1) * MAP_SIZE, + BufferedImage.TYPE_INT_RGB + ) + + connection.prepareStatement( + """ + SELECT n.name, g.data + FROM resolved_groups g + JOIN names n ON n.hash = g.name_hash + WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND + n.name ~ '^m(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$' + """.trimIndent() + ).use { stmt -> + stmt.setInt(1, masterIndexId) + + stmt.executeQuery().use { rows -> + while (rows.next()) { + val name = rows.getString(1) + val bytes = rows.getBytes(2) + + val (x, z) = getMapCoordinates(name) + + Unpooled.wrappedBuffer(bytes).use { compressed -> + Js5Compression.uncompress(compressed).use { uncompressed -> + renderMap(image, x - x0, z - z0, uncompressed, underlayColors, overlayColors) + } + } + } + } + } + + // render state overlay + val graphics = image.createGraphics() + + for (x in x0..x1) { + for (z in z0..z1) { + val mapSquare = getMapSquare(x, z) + val state = states[mapSquare] ?: continue + + renderStateOverlay(image, graphics, x - x0, z - z0, state) + } + } + + return@execute image + } + } + + private fun readIndex(connection: Connection, masterIndexId: Int, archiveId: Int): Js5Index? { + connection.prepareStatement( + """ + SELECT data + FROM resolved_indexes + WHERE master_index_id = ? AND archive_id = ? + """.trimIndent() + ).use { stmt -> + stmt.setInt(1, masterIndexId) + stmt.setInt(2, archiveId) + + stmt.executeQuery().use { rows -> + if (!rows.next()) { + return null + } + + val bytes = rows.getBytes(1) + + Unpooled.wrappedBuffer(bytes).use { compressed -> + Js5Compression.uncompress(compressed).use { uncompressed -> + return Js5Index.read(uncompressed) + } + } + } + } + } + + private fun readGroup( + connection: Connection, + masterIndexId: Int, + archiveId: Int, + group: Js5Index.Group + ): Int2ObjectSortedMap? { + connection.prepareStatement( + """ + SELECT data + FROM resolved_groups + WHERE master_index_id = ? AND archive_id = ? AND group_id = ? + """.trimIndent() + ).use { stmt -> + stmt.setInt(1, masterIndexId) + stmt.setInt(2, archiveId) + stmt.setInt(3, group.id) + + stmt.executeQuery().use { rows -> + if (!rows.next()) { + return null + } + + val bytes = rows.getBytes(1) + + Unpooled.wrappedBuffer(bytes).use { compressed -> + Js5Compression.uncompress(compressed).use { uncompressed -> + return Group.unpack(uncompressed, group) + } + } + } + } + } + + private fun createOverlayColors(overlays: Map, textures: Map): Map { + return overlays.mapValues { (_, type) -> + if (type.blendColor != -1) { + type.blendColor + } else if (type.texture != -1 && type.texture in textures) { + val averageColor = textures[type.texture]!! + Colors.hslToRgb(Colors.multiplyLightness(averageColor, 96)) + } else if (type.color == 0xFF00FF) { + 0 + } else { + type.color + } + } + } + + private fun renderMap( + image: BufferedImage, + x: Int, + z: Int, + buf: ByteBuf, + underlayColors: Map, + overlayColors: Map + ) { + for (plane in 0 until LEVELS) { + for (dx in 0 until MAP_SIZE) { + for (dz in 0 until MAP_SIZE) { + var overlay = 0 + var shape = 0 + var underlay = 0 + + while (true) { + val code = buf.readUnsignedByte().toInt() + if (code == 0) { + break + } else if (code == 1) { + buf.skipBytes(1) + break + } else if (code <= 49) { + overlay = buf.readUnsignedByte().toInt() + shape = (code - 2) shr 2 + } else if (code <= 81) { + // empty + } else { + underlay = code - 81 + } + } + + var color = 0 + + if (underlay != 0) { + color = underlayColors[underlay - 1]!! + } + + if (overlay != 0 && shape == 0) { + color = overlayColors[overlay - 1]!! + } + + if (color > 0) { + image.setRGB(x * MAP_SIZE + dx, image.height - (z * MAP_SIZE + dz) - 1, color) + } + } + } + } + } + + private fun renderStateOverlay( + image: BufferedImage, + graphics: Graphics2D, + mapX: Int, + mapZ: Int, + state: MapSquareState + ) { + val x = mapX * MAP_SIZE + val y = image.height - (mapZ + 1) * MAP_SIZE + + if (state != MapSquareState.VALID) { + graphics.color = state.fillColor + graphics.fillRect( + x, + y, + MAP_SIZE, + MAP_SIZE + ) + } + + graphics.color = state.outlineColor + graphics.drawRect( + x, + y, + MAP_SIZE - 1, + MAP_SIZE - 1 + ) + + val label = "${mapX}_$mapZ" + + val labelWidth = graphics.fontMetrics.stringWidth(label) + val labelHeight = graphics.fontMetrics.height + val labelAscent = graphics.fontMetrics.ascent + + val labelX = x + (MAP_SIZE - labelWidth) / 2 + val labelY = y + (MAP_SIZE - labelHeight) / 2 + labelAscent + + graphics.color = Color.BLACK + graphics.drawString(label, labelX + 1, labelY) + + graphics.color = Color.BLACK + graphics.drawString(label, labelX, labelY + 1) + + graphics.color = Color.WHITE + graphics.drawString(label, labelX, labelY) + } + + private companion object { + private val LOC_OR_MAP_NAME_REGEX = Regex("[lm](\\d+)_(\\d+)") + private const val MAP_SIZE = 64 + private const val LEVELS = 4 + + private fun getMapCoordinates(name: String): Pair { + val match = LOC_OR_MAP_NAME_REGEX.matchEntire(name) + require(match != null) + + val x = match.groupValues[1].toInt() + val z = match.groupValues[2].toInt() + + return Pair(x, z) + } + + private fun getMapSquare(x: Int, z: Int): Int { + return (x shl 8) or z + } + } +} diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt index 9e28c323..3d6435ea 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt @@ -10,21 +10,28 @@ import io.ktor.response.respond import io.ktor.response.respondOutputStream import io.ktor.thymeleaf.ThymeleafContent import io.netty.buffer.ByteBufAllocator +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import org.openrs2.archive.cache.CacheExporter +import org.openrs2.archive.map.MapRenderer import org.openrs2.cache.DiskStoreZipWriter import java.nio.file.attribute.FileTime import java.time.Instant import java.util.zip.Deflater import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream +import javax.imageio.ImageIO import javax.inject.Inject import javax.inject.Singleton @Singleton public class CachesController @Inject constructor( private val exporter: CacheExporter, + private val renderer: MapRenderer, private val alloc: ByteBufAllocator ) { + private val renderSemaphore = Semaphore(1) + public suspend fun index(call: ApplicationCall) { val caches = exporter.list() call.respond(ThymeleafContent("caches/index.html", mapOf("caches" to caches))) @@ -128,4 +135,25 @@ public class CachesController @Inject constructor( } } } + + public suspend fun renderMap(call: ApplicationCall) { + val id = call.parameters["id"]?.toIntOrNull() + if (id == null) { + call.respond(HttpStatusCode.NotFound) + return + } + + /* + * The temporary BufferedImages used by the MapRenderer use a large + * amount of heap space. We limit the number of renders that can be + * performed in parallel to prevent OOMs. + */ + renderSemaphore.withPermit { + val image = renderer.render(id) + + call.respondOutputStream(contentType = ContentType.Image.PNG) { + ImageIO.write(image, "PNG", this) + } + } + } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt index cc38f952..6bbfbfe3 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt @@ -67,6 +67,7 @@ public class WebServer @Inject constructor( } get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) } get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) } + get("/caches/{id}/map.png") { cachesController.renderMap(call) } static("/static") { resources("/org/openrs2/archive/static") } } }.start(wait = true) diff --git a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql index 063b27a9..b121b836 100644 --- a/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql +++ b/archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql @@ -190,15 +190,15 @@ FROM master_indexes m JOIN master_index_archives a ON a.master_index_id = m.id JOIN resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE; -CREATE VIEW resolved_groups (master_index_id, archive_id, group_id, name_hash, version, data, key_id) AS +CREATE VIEW resolved_groups (master_index_id, archive_id, group_id, name_hash, version, data, encrypted, empty_loc, key_id) AS WITH i AS NOT MATERIALIZED ( SELECT master_index_id, archive_id, data, container_id FROM resolved_indexes ) -SELECT i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, NULL +SELECT i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, FALSE, FALSE, NULL FROM i UNION ALL -SELECT i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.key_id +SELECT i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.encrypted, c.empty_loc, c.key_id FROM i JOIN index_groups ig ON ig.container_id = i.container_id JOIN resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE; diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index a1d4b244..bc09f640 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -83,6 +83,8 @@ class="dropdown-item">Keys (JSON)
  • Keys (Text)
    • +
  • Map
    • Keys (JSON) Keys (Text) + Map