From 6023569ce0d9e4dbc3f19fafea8193049836c814 Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 2 Jan 2021 13:07:39 +0000 Subject: [PATCH] Add code for packing and unpacking groups Signed-off-by: Graham --- .../main/kotlin/org/openrs2/cache/Group.kt | 99 ++++++++++ .../kotlin/org/openrs2/cache/GroupTest.kt | 180 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 cache/src/main/kotlin/org/openrs2/cache/Group.kt create mode 100644 cache/src/test/kotlin/org/openrs2/cache/GroupTest.kt diff --git a/cache/src/main/kotlin/org/openrs2/cache/Group.kt b/cache/src/main/kotlin/org/openrs2/cache/Group.kt new file mode 100644 index 00000000..7aa923cc --- /dev/null +++ b/cache/src/main/kotlin/org/openrs2/cache/Group.kt @@ -0,0 +1,99 @@ +package org.openrs2.cache + +import io.netty.buffer.ByteBuf +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMap +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMaps +import org.openrs2.buffer.use + +public object Group { + public fun unpack(input: ByteBuf, group: Js5Index.Group): Int2ObjectSortedMap { + require(group.size >= 1) + + val singleEntry = group.singleOrNull() + if (singleEntry != null) { + return Int2ObjectSortedMaps.singleton(singleEntry.id, input.retain()) + } + + require(input.isReadable) + + val stripes = input.getUnsignedByte(input.writerIndex() - 1) + + var dataIndex = input.readerIndex() + val trailerIndex = input.writerIndex() - (stripes * group.size * 4) - 1 + require(trailerIndex >= dataIndex) + + input.readerIndex(trailerIndex) + + val lens = IntArray(group.size) + for (i in 0 until stripes) { + var prevLen = 0 + for (j in lens.indices) { + prevLen += input.readInt() + lens[j] += prevLen + } + } + + input.readerIndex(trailerIndex) + + val files = Int2ObjectAVLTreeMap() + try { + for ((i, entry) in group.withIndex()) { + files[entry.id] = input.alloc().buffer(lens[i], lens[i]) + } + + for (i in 0 until stripes) { + var prevLen = 0 + for (entry in group) { + prevLen += input.readInt() + input.getBytes(dataIndex, files[entry.id], prevLen) + dataIndex += prevLen + } + } + + check(dataIndex == trailerIndex) + + // consume stripes byte too + input.skipBytes(1) + check(!input.isReadable) + + files.values.forEach(ByteBuf::retain) + return files + } finally { + files.values.forEach(ByteBuf::release) + } + } + + // TODO(gpe): support multiple stripes (tricky, as the best sizes are + // probably specific to the format we're packing...) + public fun pack(files: Int2ObjectSortedMap): ByteBuf { + require(files.isNotEmpty()) + + val first = files.values.first() + if (files.size == 1) { + return first + } + + first.alloc().buffer().use { output -> + if (files.values.all { !it.isReadable }) { + output.writeByte(0) + return output.retain() + } + + for (file in files.values) { + output.writeBytes(file, file.readerIndex(), file.readableBytes()) + } + + var prevLen = 0 + for (file in files.values) { + val len = file.readableBytes() + output.writeInt(len - prevLen) + prevLen = len + } + + output.writeByte(1) + + return output.retain() + } + } +} diff --git a/cache/src/test/kotlin/org/openrs2/cache/GroupTest.kt b/cache/src/test/kotlin/org/openrs2/cache/GroupTest.kt new file mode 100644 index 00000000..014a4228 --- /dev/null +++ b/cache/src/test/kotlin/org/openrs2/cache/GroupTest.kt @@ -0,0 +1,180 @@ +package org.openrs2.cache + +import io.netty.buffer.ByteBuf +import io.netty.buffer.Unpooled +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap +import it.unimi.dsi.fastutil.ints.Int2ObjectSortedMaps +import org.openrs2.buffer.use +import org.openrs2.buffer.wrappedBuffer +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +object GroupTest { + private val index = Js5Index(Js5Protocol.ORIGINAL) + private val zeroFiles = index.createOrGet(0) + private val oneFile = index.createOrGet(1).apply { + createOrGet(1) + } + private val multipleFiles = index.createOrGet(2).apply { + createOrGet(0) + createOrGet(1) + createOrGet(3) + } + + @Test + fun testPackEmpty() { + assertFailsWith { + Group.pack(Int2ObjectSortedMaps.emptyMap()).release() + } + } + + @Test + fun testUnpackEmpty() { + assertFailsWith { + val files = Group.unpack(Unpooled.EMPTY_BUFFER, zeroFiles) + files.values.forEach(ByteBuf::release) + } + + assertFailsWith { + val files = Group.unpack(Unpooled.EMPTY_BUFFER, multipleFiles) + files.values.forEach(ByteBuf::release) + } + } + + @Test + fun testPackSingle() { + wrappedBuffer(0, 1, 2, 3).use { buf -> + assertEquals(buf, Group.pack(Int2ObjectSortedMaps.singleton(1, buf))) + } + } + + @Test + fun testUnpackSingle() { + wrappedBuffer(0, 1, 2, 3).use { buf -> + val expected = Int2ObjectSortedMaps.singleton(1, buf) + val actual = Group.unpack(buf.slice(), oneFile) + try { + assertEquals(expected, actual) + } finally { + actual.values.forEach(ByteBuf::release) + } + } + } + + @Test + fun testPackZeroStripes() { + val files = Int2ObjectAVLTreeMap() + files[0] = Unpooled.EMPTY_BUFFER + files[1] = Unpooled.EMPTY_BUFFER + files[3] = Unpooled.EMPTY_BUFFER + + wrappedBuffer(0).use { expected -> + Group.pack(files).use { actual -> + assertEquals(expected, actual) + } + } + } + + @Test + fun testUnpackZeroStripes() { + val expected = Int2ObjectAVLTreeMap() + expected[0] = Unpooled.EMPTY_BUFFER + expected[1] = Unpooled.EMPTY_BUFFER + expected[3] = Unpooled.EMPTY_BUFFER + + wrappedBuffer(0).use { buf -> + val actual = Group.unpack(buf, multipleFiles) + try { + assertEquals(expected, actual) + } finally { + actual.values.forEach(ByteBuf::release) + } + } + } + + @Test + fun testPackOneStripe() { + val files = Int2ObjectAVLTreeMap() + try { + files[0] = wrappedBuffer(0, 1, 2) + files[1] = wrappedBuffer(3, 4, 5, 6, 7) + files[3] = wrappedBuffer(8, 9) + + wrappedBuffer( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 0, 0, 3, + 0, 0, 0, 2, + 0xFF.toByte(), 0xFF.toByte(), 0xFF.toByte(), 0xFD.toByte(), + 1 + ).use { expected -> + Group.pack(files).use { actual -> + assertEquals(expected, actual) + } + } + } finally { + files.values.forEach(ByteBuf::release) + } + } + + @Test + fun testUnpackOneStripe() { + val expected = Int2ObjectAVLTreeMap() + try { + expected[0] = wrappedBuffer(0, 1, 2) + expected[1] = wrappedBuffer(3, 4, 5, 6, 7) + expected[3] = wrappedBuffer(8, 9) + + wrappedBuffer( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 0, 0, 0, 3, + 0, 0, 0, 2, + 0xFF.toByte(), 0xFF.toByte(), 0xFF.toByte(), 0xFD.toByte(), + 1 + ).use { buf -> + val actual = Group.unpack(buf, multipleFiles) + try { + assertEquals(expected, actual) + } finally { + actual.values.forEach(ByteBuf::release) + } + } + } finally { + expected.values.forEach(ByteBuf::release) + } + } + + @Test + fun testUnpackMultipleStripes() { + val expected = Int2ObjectAVLTreeMap() + try { + expected[0] = wrappedBuffer(0, 1, 2) + expected[1] = wrappedBuffer(3, 4, 5, 6, 7) + expected[3] = wrappedBuffer(8, 9) + + wrappedBuffer( + 0, 1, + 3, 4, + 8, 9, + 2, + 5, 6, 7, + 0, 0, 0, 2, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 1, + 0, 0, 0, 2, + 0xFF.toByte(), 0xFF.toByte(), 0xFF.toByte(), 0xFD.toByte(), + 2 + ).use { buf -> + val actual = Group.unpack(buf, multipleFiles) + try { + assertEquals(expected, actual) + } finally { + actual.values.forEach(ByteBuf::release) + } + } + } finally { + expected.values.forEach(ByteBuf::release) + } + } +}