Add initial support for plotting a map of valid/invalid XTEA keys

Signed-off-by: Graham <gpe@openrs2.org>
bzip2
Graham 4 years ago
parent dbd169aa1a
commit 93ee863e20
  1. 107
      archive/src/main/kotlin/org/openrs2/archive/map/Colors.kt
  2. 57
      archive/src/main/kotlin/org/openrs2/archive/map/FloType.kt
  3. 34
      archive/src/main/kotlin/org/openrs2/archive/map/FluType.kt
  4. 426
      archive/src/main/kotlin/org/openrs2/archive/map/MapRenderer.kt
  5. 28
      archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt
  6. 1
      archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt
  7. 6
      archive/src/main/resources/org/openrs2/archive/migrations/V1__init.sql
  8. 2
      archive/src/main/resources/org/openrs2/archive/templates/caches/index.html
  9. 2
      archive/src/main/resources/org/openrs2/archive/templates/caches/show.html

@ -0,0 +1,107 @@
package org.openrs2.archive.map
import kotlin.math.max
import kotlin.math.min
import kotlin.math.pow
public object Colors {
private val HSL_TO_RGB = IntArray(65536)
private const val BRIGHTNESS = 0.8
init {
var i = 0
for (h in 0 until 64) {
for (s in 0 until 8) {
for (l in 0 until 128) {
val hue = h.toDouble() / 64 + 0.0078125
val saturation = s.toDouble() / 8 + 0.0625
val lightness = l.toDouble() / 128
HSL_TO_RGB[i++] = hslToRgb(hue, saturation, lightness)
}
}
}
}
private fun hslToRgb(h: Double, s: Double, l: Double): Int {
var r = l
var g = l
var b = l
if (s != 0.0) {
val q = if (l * 2 < 1) {
l * (s + 1)
} else {
l + s - (l * s)
}
val p = l * 2 - q
var tr = h + (1.0 / 3)
if (tr > 1) {
tr--
}
var tb = h - (1.0 / 3)
if (tb < 0) {
tb++
}
r = if (tr * 6 < 1) {
tr * (q - p) * 6 + p
} else if (tr * 2 < 1) {
q
} else if (tr * 3 < 2) {
(2.0 / 3 - tr) * (q - p) * 6 + p
} else {
p
}
g = if (h * 6 < 1) {
h * (q - p) * 6 + p
} else if (h * 2 < 1) {
q
} else if (h * 3 < 2) {
(2.0 / 3 - h) * (q - p) * 6 + p
} else {
p
}
b = if (tb * 6 < 1) {
tb * (q - p) * 6 + p
} else if (tb * 2 < 1) {
q
} else if (tb * 3 < 2) {
(2.0 / 3 - tb) * (q - p) * 6 + p
} else {
p
}
}
val red = (r.pow(BRIGHTNESS) * 256).toInt()
val green = (g.pow(BRIGHTNESS) * 256).toInt()
val blue = (b.pow(BRIGHTNESS) * 256).toInt()
var rgb = (red shl 16) or (green shl 8) or blue
if (rgb == 0) {
rgb = 1
}
return rgb
}
public fun hslToRgb(hsl: Int): Int {
return HSL_TO_RGB[hsl]
}
public fun multiplyLightness(hsl: Int, factor: Int): Int {
return when (hsl) {
-2 -> 12345678
-1 -> 127 - min(max(factor, 0), 127)
else -> {
var l = ((hsl and 0x7F) * factor) shr 7
l = min(max(l, 2), 126)
(hsl and 0xFF80) or l
}
}
}
}

@ -0,0 +1,57 @@
package org.openrs2.archive.map
import io.netty.buffer.ByteBuf
public data class FloType(
var color: Int = 0,
var texture: Int = -1,
var blendColor: Int = -1
) {
public companion object {
public fun read(buf: ByteBuf): FloType {
val type = FloType()
while (true) {
val code = buf.readUnsignedByte().toInt()
if (code == 0) {
break
} else if (code == 1) {
type.color = buf.readUnsignedMedium()
} else if (code == 2) {
type.texture = buf.readUnsignedByte().toInt()
} else if (code == 3) {
type.texture = buf.readUnsignedShort()
if (type.texture == 65535) {
type.texture = -1
}
} else if (code == 5) {
// empty
} else if (code == 7) {
type.blendColor = buf.readUnsignedMedium()
} else if (code == 8) {
// empty
} else if (code == 9) {
buf.skipBytes(2)
} else if (code == 10) {
// empty
} else if (code == 11) {
buf.skipBytes(1)
} else if (code == 12) {
// empty
} else if (code == 13) {
buf.skipBytes(3)
} else if (code == 14) {
buf.skipBytes(1)
} else if (code == 15) {
buf.skipBytes(2)
} else if (code == 16) {
buf.skipBytes(1)
} else {
throw IllegalArgumentException("Unsupported code: $code")
}
}
return type
}
}
}

@ -0,0 +1,34 @@
package org.openrs2.archive.map
import io.netty.buffer.ByteBuf
public data class FluType(
var color: Int = 0
) {
public companion object {
public fun read(buf: ByteBuf): FluType {
val type = FluType()
while (true) {
val code = buf.readUnsignedByte().toInt()
if (code == 0) {
break
} else if (code == 1) {
type.color = buf.readUnsignedMedium()
} else if (code == 2) {
buf.skipBytes(2)
} else if (code == 3) {
buf.skipBytes(2)
} else if (code == 4) {
// empty
} else if (code == 5) {
// empty
} else {
throw IllegalArgumentException("Unsupported code: $code")
}
}
return type
}
}
}

@ -0,0 +1,426 @@
package org.openrs2.archive.map
import io.netty.buffer.ByteBuf
import io.netty.buffer.Unpooled
import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap
import org.openrs2.buffer.use
import org.openrs2.cache.Group
import org.openrs2.cache.Js5Archive
import org.openrs2.cache.Js5Compression
import org.openrs2.cache.Js5ConfigGroup
import org.openrs2.cache.Js5Index
import org.openrs2.db.Database
import java.awt.Color
import java.awt.Graphics2D
import java.awt.image.BufferedImage
import java.sql.Connection
import javax.inject.Inject
import kotlin.math.max
import kotlin.math.min
public class MapRenderer @Inject constructor(
private val database: Database
) {
private enum class MapSquareState(
val outlineColor: Color
) {
UNKNOWN(Color.RED),
EMPTY(Color.GRAY),
VALID(Color.GREEN);
val fillColor = Color(outlineColor.red, outlineColor.green, outlineColor.blue, 128)
}
public suspend fun render(masterIndexId: Int): BufferedImage {
return database.execute { connection ->
// read config index
val configIndex = readIndex(connection, masterIndexId, Js5Archive.CONFIG)
?: throw IllegalArgumentException("Config index missing")
// read FluType group
val underlayColors = mutableMapOf<Int, Int>()
val underlayGroup = configIndex[Js5ConfigGroup.FLUTYPE]
?: throw IllegalArgumentException("FluType group missing in index")
val underlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, underlayGroup)
?: throw IllegalArgumentException("FluType group missing")
try {
for ((id, file) in underlayFiles) {
underlayColors[id] = FluType.read(file).color
}
} finally {
underlayFiles.values.forEach(ByteBuf::release)
}
// read FloType group
val overlays = mutableMapOf<Int, FloType>()
val overlayGroup = configIndex[Js5ConfigGroup.FLOTYPE]
?: throw IllegalArgumentException("FloType group missing in index")
val overlayFiles = readGroup(connection, masterIndexId, Js5Archive.CONFIG, overlayGroup)
?: throw IllegalArgumentException("FloType group missing")
try {
for ((id, file) in overlayFiles) {
overlays[id] = FloType.read(file)
}
} finally {
overlayFiles.values.forEach(ByteBuf::release)
}
// read textures
val textures = mutableMapOf<Int, Int>()
val materialsIndex = readIndex(connection, masterIndexId, Js5Archive.MATERIALS)
if (materialsIndex != null) {
val materialsGroup = materialsIndex[0]
?: throw IllegalArgumentException("Materials group missing in index")
val materialsFiles = readGroup(connection, masterIndexId, Js5Archive.MATERIALS, materialsGroup)
?: throw IllegalArgumentException("Materials group missing")
try {
val metadata = materialsFiles[0]
val len = metadata.readUnsignedShort()
val ids = mutableSetOf<Int>()
for (id in 0 until len) {
if (metadata.readBoolean()) {
ids += id
}
}
val use = mutableSetOf<Int>()
for (id in ids) {
if (metadata.readBoolean()) {
use += id
}
}
metadata.skipBytes(ids.size * 7)
for (id in ids) {
textures[id] = metadata.readUnsignedShort()
if (id !in use) {
textures.remove(id)
}
}
} finally {
materialsFiles.values.forEach(ByteBuf::release)
}
} else {
val textureIndex = readIndex(connection, masterIndexId, Js5Archive.TEXTURES)
?: throw IllegalArgumentException("Textures index missing")
val textureGroup = textureIndex[0]
?: throw IllegalArgumentException("Textures group missing from index")
val textureFiles = readGroup(connection, masterIndexId, Js5Archive.TEXTURES, textureGroup)
?: throw IllegalArgumentException("Textures group missing")
try {
for ((id, file) in textureFiles) {
textures[id] = file.readUnsignedShort()
}
} finally {
textureFiles.values.forEach(ByteBuf::release)
}
}
// create overlay colors
val overlayColors = createOverlayColors(overlays, textures)
// read loc encrypted/empty loc flags and keys and determine bounds of the map
var x0 = Int.MAX_VALUE
var x1 = Int.MIN_VALUE
var z0 = Int.MAX_VALUE
var z1 = Int.MIN_VALUE
val states = mutableMapOf<Int, MapSquareState>()
connection.prepareStatement(
"""
SELECT n.name, g.encrypted, g.empty_loc, g.key_id
FROM resolved_groups g
JOIN names n ON n.hash = g.name_hash
WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND
n.name ~ '^[lm](?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'
""".trimIndent()
).use { stmt ->
stmt.setInt(1, masterIndexId)
stmt.executeQuery().use { rows ->
while (rows.next()) {
val name = rows.getString(1)
val encrypted = rows.getBoolean(2)
val empty = rows.getBoolean(3)
var keyId: Long? = rows.getLong(4)
if (rows.wasNull()) {
keyId = null
}
val (x, z) = getMapCoordinates(name)
x0 = min(x0, x)
x1 = max(x1, x)
z0 = min(z0, z)
z1 = max(z1, z)
if (name.startsWith('l')) {
val mapSquare = getMapSquare(x, z)
states[mapSquare] = if (!encrypted || keyId != null) {
MapSquareState.VALID
} else if (empty) {
MapSquareState.EMPTY
} else {
MapSquareState.UNKNOWN
}
}
}
}
}
if (x0 == Int.MAX_VALUE) {
throw IllegalArgumentException("Map empty")
}
// read and render maps
val image = BufferedImage(
((x1 - x0) + 1) * MAP_SIZE,
((z1 - z0) + 1) * MAP_SIZE,
BufferedImage.TYPE_INT_RGB
)
connection.prepareStatement(
"""
SELECT n.name, g.data
FROM resolved_groups g
JOIN names n ON n.hash = g.name_hash
WHERE g.master_index_id = ? AND g.archive_id = ${Js5Archive.MAPS} AND
n.name ~ '^m(?:[0-9]|[1-9][0-9])_(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$'
""".trimIndent()
).use { stmt ->
stmt.setInt(1, masterIndexId)
stmt.executeQuery().use { rows ->
while (rows.next()) {
val name = rows.getString(1)
val bytes = rows.getBytes(2)
val (x, z) = getMapCoordinates(name)
Unpooled.wrappedBuffer(bytes).use { compressed ->
Js5Compression.uncompress(compressed).use { uncompressed ->
renderMap(image, x - x0, z - z0, uncompressed, underlayColors, overlayColors)
}
}
}
}
}
// render state overlay
val graphics = image.createGraphics()
for (x in x0..x1) {
for (z in z0..z1) {
val mapSquare = getMapSquare(x, z)
val state = states[mapSquare] ?: continue
renderStateOverlay(image, graphics, x - x0, z - z0, state)
}
}
return@execute image
}
}
private fun readIndex(connection: Connection, masterIndexId: Int, archiveId: Int): Js5Index? {
connection.prepareStatement(
"""
SELECT data
FROM resolved_indexes
WHERE master_index_id = ? AND archive_id = ?
""".trimIndent()
).use { stmt ->
stmt.setInt(1, masterIndexId)
stmt.setInt(2, archiveId)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
return null
}
val bytes = rows.getBytes(1)
Unpooled.wrappedBuffer(bytes).use { compressed ->
Js5Compression.uncompress(compressed).use { uncompressed ->
return Js5Index.read(uncompressed)
}
}
}
}
}
private fun readGroup(
connection: Connection,
masterIndexId: Int,
archiveId: Int,
group: Js5Index.Group
): Int2ObjectSortedMap<ByteBuf>? {
connection.prepareStatement(
"""
SELECT data
FROM resolved_groups
WHERE master_index_id = ? AND archive_id = ? AND group_id = ?
""".trimIndent()
).use { stmt ->
stmt.setInt(1, masterIndexId)
stmt.setInt(2, archiveId)
stmt.setInt(3, group.id)
stmt.executeQuery().use { rows ->
if (!rows.next()) {
return null
}
val bytes = rows.getBytes(1)
Unpooled.wrappedBuffer(bytes).use { compressed ->
Js5Compression.uncompress(compressed).use { uncompressed ->
return Group.unpack(uncompressed, group)
}
}
}
}
}
private fun createOverlayColors(overlays: Map<Int, FloType>, textures: Map<Int, Int>): Map<Int, Int> {
return overlays.mapValues { (_, type) ->
if (type.blendColor != -1) {
type.blendColor
} else if (type.texture != -1 && type.texture in textures) {
val averageColor = textures[type.texture]!!
Colors.hslToRgb(Colors.multiplyLightness(averageColor, 96))
} else if (type.color == 0xFF00FF) {
0
} else {
type.color
}
}
}
private fun renderMap(
image: BufferedImage,
x: Int,
z: Int,
buf: ByteBuf,
underlayColors: Map<Int, Int>,
overlayColors: Map<Int, Int>
) {
for (plane in 0 until LEVELS) {
for (dx in 0 until MAP_SIZE) {
for (dz in 0 until MAP_SIZE) {
var overlay = 0
var shape = 0
var underlay = 0
while (true) {
val code = buf.readUnsignedByte().toInt()
if (code == 0) {
break
} else if (code == 1) {
buf.skipBytes(1)
break
} else if (code <= 49) {
overlay = buf.readUnsignedByte().toInt()
shape = (code - 2) shr 2
} else if (code <= 81) {
// empty
} else {
underlay = code - 81
}
}
var color = 0
if (underlay != 0) {
color = underlayColors[underlay - 1]!!
}
if (overlay != 0 && shape == 0) {
color = overlayColors[overlay - 1]!!
}
if (color > 0) {
image.setRGB(x * MAP_SIZE + dx, image.height - (z * MAP_SIZE + dz) - 1, color)
}
}
}
}
}
private fun renderStateOverlay(
image: BufferedImage,
graphics: Graphics2D,
mapX: Int,
mapZ: Int,
state: MapSquareState
) {
val x = mapX * MAP_SIZE
val y = image.height - (mapZ + 1) * MAP_SIZE
if (state != MapSquareState.VALID) {
graphics.color = state.fillColor
graphics.fillRect(
x,
y,
MAP_SIZE,
MAP_SIZE
)
}
graphics.color = state.outlineColor
graphics.drawRect(
x,
y,
MAP_SIZE - 1,
MAP_SIZE - 1
)
val label = "${mapX}_$mapZ"
val labelWidth = graphics.fontMetrics.stringWidth(label)
val labelHeight = graphics.fontMetrics.height
val labelAscent = graphics.fontMetrics.ascent
val labelX = x + (MAP_SIZE - labelWidth) / 2
val labelY = y + (MAP_SIZE - labelHeight) / 2 + labelAscent
graphics.color = Color.BLACK
graphics.drawString(label, labelX + 1, labelY)
graphics.color = Color.BLACK
graphics.drawString(label, labelX, labelY + 1)
graphics.color = Color.WHITE
graphics.drawString(label, labelX, labelY)
}
private companion object {
private val LOC_OR_MAP_NAME_REGEX = Regex("[lm](\\d+)_(\\d+)")
private const val MAP_SIZE = 64
private const val LEVELS = 4
private fun getMapCoordinates(name: String): Pair<Int, Int> {
val match = LOC_OR_MAP_NAME_REGEX.matchEntire(name)
require(match != null)
val x = match.groupValues[1].toInt()
val z = match.groupValues[2].toInt()
return Pair(x, z)
}
private fun getMapSquare(x: Int, z: Int): Int {
return (x shl 8) or z
}
}
}

@ -10,21 +10,28 @@ import io.ktor.response.respond
import io.ktor.response.respondOutputStream import io.ktor.response.respondOutputStream
import io.ktor.thymeleaf.ThymeleafContent import io.ktor.thymeleaf.ThymeleafContent
import io.netty.buffer.ByteBufAllocator import io.netty.buffer.ByteBufAllocator
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import org.openrs2.archive.cache.CacheExporter import org.openrs2.archive.cache.CacheExporter
import org.openrs2.archive.map.MapRenderer
import org.openrs2.cache.DiskStoreZipWriter import org.openrs2.cache.DiskStoreZipWriter
import java.nio.file.attribute.FileTime import java.nio.file.attribute.FileTime
import java.time.Instant import java.time.Instant
import java.util.zip.Deflater import java.util.zip.Deflater
import java.util.zip.ZipEntry import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream import java.util.zip.ZipOutputStream
import javax.imageio.ImageIO
import javax.inject.Inject import javax.inject.Inject
import javax.inject.Singleton import javax.inject.Singleton
@Singleton @Singleton
public class CachesController @Inject constructor( public class CachesController @Inject constructor(
private val exporter: CacheExporter, private val exporter: CacheExporter,
private val renderer: MapRenderer,
private val alloc: ByteBufAllocator private val alloc: ByteBufAllocator
) { ) {
private val renderSemaphore = Semaphore(1)
public suspend fun index(call: ApplicationCall) { public suspend fun index(call: ApplicationCall) {
val caches = exporter.list() val caches = exporter.list()
call.respond(ThymeleafContent("caches/index.html", mapOf("caches" to caches))) call.respond(ThymeleafContent("caches/index.html", mapOf("caches" to caches)))
@ -128,4 +135,25 @@ public class CachesController @Inject constructor(
} }
} }
} }
public suspend fun renderMap(call: ApplicationCall) {
val id = call.parameters["id"]?.toIntOrNull()
if (id == null) {
call.respond(HttpStatusCode.NotFound)
return
}
/*
* The temporary BufferedImages used by the MapRenderer use a large
* amount of heap space. We limit the number of renders that can be
* performed in parallel to prevent OOMs.
*/
renderSemaphore.withPermit {
val image = renderer.render(id)
call.respondOutputStream(contentType = ContentType.Image.PNG) {
ImageIO.write(image, "PNG", this)
}
}
}
} }

@ -67,6 +67,7 @@ public class WebServer @Inject constructor(
} }
get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) } get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) }
get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) } get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) }
get("/caches/{id}/map.png") { cachesController.renderMap(call) }
static("/static") { resources("/org/openrs2/archive/static") } static("/static") { resources("/org/openrs2/archive/static") }
} }
}.start(wait = true) }.start(wait = true)

@ -190,15 +190,15 @@ FROM master_indexes m
JOIN master_index_archives a ON a.master_index_id = m.id JOIN master_index_archives a ON a.master_index_id = m.id
JOIN resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE; JOIN resolve_index(a.archive_id, a.crc32, a.version) c ON TRUE;
CREATE VIEW resolved_groups (master_index_id, archive_id, group_id, name_hash, version, data, key_id) AS CREATE VIEW resolved_groups (master_index_id, archive_id, group_id, name_hash, version, data, encrypted, empty_loc, key_id) AS
WITH i AS NOT MATERIALIZED ( WITH i AS NOT MATERIALIZED (
SELECT master_index_id, archive_id, data, container_id SELECT master_index_id, archive_id, data, container_id
FROM resolved_indexes FROM resolved_indexes
) )
SELECT i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, NULL SELECT i.master_index_id, 255::uint1, i.archive_id::INTEGER, NULL, NULL, i.data, FALSE, FALSE, NULL
FROM i FROM i
UNION ALL UNION ALL
SELECT i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.key_id SELECT i.master_index_id, i.archive_id, ig.group_id, ig.name_hash, ig.version, c.data, c.encrypted, c.empty_loc, c.key_id
FROM i FROM i
JOIN index_groups ig ON ig.container_id = i.container_id JOIN index_groups ig ON ig.container_id = i.container_id
JOIN resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE; JOIN resolve_group(i.archive_id, ig.group_id, ig.crc32, ig.version) c ON TRUE;

@ -83,6 +83,8 @@
class="dropdown-item">Keys (JSON)</a></li> class="dropdown-item">Keys (JSON)</a></li>
<li><a th:href="${'/caches/' + cache.id + '/keys.zip'}" <li><a th:href="${'/caches/' + cache.id + '/keys.zip'}"
class="dropdown-item">Keys (Text)</a></li> class="dropdown-item">Keys (Text)</a></li>
<li><a th:href="${'/caches/' + cache.id + '/map.png'}"
class="dropdown-item">Map</a></li>
</ul> </ul>
</div> </div>
<a th:href="${'/caches/' + cache.id}" <a th:href="${'/caches/' + cache.id}"

@ -75,6 +75,8 @@
class="btn btn-primary btn-sm">Keys (JSON)</a> class="btn btn-primary btn-sm">Keys (JSON)</a>
<a th:href="${'/caches/' + cache.id + '/keys.zip'}" <a th:href="${'/caches/' + cache.id + '/keys.zip'}"
class="btn btn-primary btn-sm">Keys (Text)</a> class="btn btn-primary btn-sm">Keys (Text)</a>
<a th:href="${'/caches/' + cache.id + '/map.png'}"
class="btn btn-primary btn-sm">Map</a>
</div> </div>
</td> </td>
</tr> </tr>

Loading…
Cancel
Save