From 2d19d3f90d3fac80c008cd9300ca803283738e47 Mon Sep 17 00:00:00 2001 From: Graham Date: Sat, 29 Jan 2022 21:52:22 +0000 Subject: [PATCH] Add initial libbzip2 Java port This is still very much a work-in-progress. Signed-off-by: Graham --- README.md | 6 + compress-bzip2/LICENSE | 41 ++++ compress-bzip2/build.gradle.kts | 21 ++ .../openrs2/compress/bzip2/BitOutputStream.kt | 48 ++++ .../compress/bzip2/Bzip2OutputStream.kt | 210 ++++++++++++++++++ settings.gradle.kts | 1 + 6 files changed, 327 insertions(+) create mode 100644 compress-bzip2/LICENSE create mode 100644 compress-bzip2/build.gradle.kts create mode 100644 compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/BitOutputStream.kt create mode 100644 compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt diff --git a/README.md b/README.md index fde680d0..8abd9985 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,12 @@ OpenRS2 is available under the terms of the [ISC license][isc], which is similar to the 2-clause BSD license. The full copyright notice and terms are available in the `LICENSE` file. +The `compress-bzip2` module is derived from the reference [bzip2][bzip2] +implementation. The reference implementation's license is available in +`compress-bzip2/LICENSE`. Both this license and OpenRS2's license apply to the +derived work. + +[bzip2]: https://sourceware.org/bzip2/ [discord-badge]: https://img.shields.io/discord/684495254145335298 [discord]: https://chat.openrs2.org/ [drone-badge]: https://build.openrs2.org/api/badges/openrs2/openrs2/status.svg diff --git a/compress-bzip2/LICENSE b/compress-bzip2/LICENSE new file mode 100644 index 00000000..310f02f6 --- /dev/null +++ b/compress-bzip2/LICENSE @@ -0,0 +1,41 @@ +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.8 of 13 July 2019 + +-------------------------------------------------------------------------- diff --git a/compress-bzip2/build.gradle.kts b/compress-bzip2/build.gradle.kts new file mode 100644 index 00000000..1706b514 --- /dev/null +++ b/compress-bzip2/build.gradle.kts @@ -0,0 +1,21 @@ +plugins { + `maven-publish` + kotlin("jvm") +} + +publishing { + publications.create("maven") { + from(components["java"]) + + pom { + packaging = "jar" + name.set("OpenRS2 bzip2") + description.set( + """ + Provides a bzip2 encoder that produces output identical to the + reference implementation. + """.trimIndent() + ) + } + } +} diff --git a/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/BitOutputStream.kt b/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/BitOutputStream.kt new file mode 100644 index 00000000..d393ceb1 --- /dev/null +++ b/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/BitOutputStream.kt @@ -0,0 +1,48 @@ +package org.openrs2.compress.bzip2 + +import java.io.Closeable +import java.io.Flushable +import java.io.OutputStream + +public class BitOutputStream( + private val output: OutputStream +) : Flushable, Closeable { + private var buf: Int = 0 + private var live: Int = 0 + + public fun writeBits(n: Int, v: Int) { + while (live >= 8) { + output.write(buf ushr 24) + buf = buf shl 8 + live -= 8 + } + + buf = buf or (v shl (32 - live - n)) + live += n + } + + public fun writeBoolean(v: Boolean) { + writeBits(1, if (v) 1 else 0) + } + + public fun writeByte(v: Int) { + writeBits(8, v) + } + + public fun writeInt(v: Int) { + writeBits(32, v) + } + + override fun flush() { + while (live > 0) { + output.write(buf ushr 24) + buf = buf shl 8 + live -= 8 + } + } + + override fun close() { + flush() + output.close() + } +} diff --git a/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt b/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt new file mode 100644 index 00000000..5a10b12d --- /dev/null +++ b/compress-bzip2/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt @@ -0,0 +1,210 @@ +package org.openrs2.compress.bzip2 + +import java.io.FilterOutputStream +import java.io.OutputStream +import java.util.zip.CRC32 + +public class Bzip2OutputStream( + output: OutputStream, + private val blockSize100k: Int, + private val workFactor: Int = DEFAULT_WORK_FACTOR +) : FilterOutputStream(output) { + // bit stream to byte stream encoder + private val output = BitOutputStream(output) + + // used to implement write(int) efficiently + private val temp = ByteArray(1) + + // current uncompressed block + private val block: ByteArray + private val blockCrc = CRC32() + private var blockSize = 0 // nblock + private var blockSizeMax: Int // nblockMAX + + private var combinedCrc = 0 + private var origPtr = 0 + private val ftab = IntArray(65537) + private val ptr: IntArray + + // run length encoder + private var stateInCh = 256 + private var stateInLen = 0 + + private val inUse = BooleanArray(256) + + init { + require(blockSize100k in 1..9) + require(workFactor in 1..250) + + val n = blockSize100k * 100_000 + block = ByteArray(n) + blockSizeMax = n - 19 + ptr = IntArray(n) + + writeHeader() + } + + override fun write(b: Int) { + temp[0] = b.toByte() + write(temp) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + // run length encode input, compress when block is almost full + for (i in off until off + len) { + val ch = b[i].toInt() and 0xFF + + if (ch != stateInCh && stateInLen == 1) { + // optimize common case + blockCrc.update(stateInCh) + inUse[stateInCh] = true + block[blockSize++] = stateInCh.toByte() + stateInCh = ch + } else if (ch != stateInCh || stateInLen == 255) { + flushRunLength() + + stateInCh = ch + stateInLen = 1 + } else { + stateInLen++ + } + + if (blockSize >= blockSizeMax) { + compressBlock() + } + } + } + + private fun flushRunLength() { + if (stateInCh == 256) { + return + } + + for (i in 0 until stateInLen) { + blockCrc.update(stateInCh) + } + + inUse[stateInCh] = true + + when (stateInLen) { + 1 -> { + block[blockSize++] = stateInCh.toByte() + } + 2 -> { + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + } + 3 -> { + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + } + else -> { + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + block[blockSize++] = stateInCh.toByte() + + val runLength = stateInLen - 4 + inUse[runLength] = true + block[blockSize++] = runLength.toByte() + } + } + } + + override fun flush() { + flushRunLength() + compressBlock() + } + + override fun close() { + flush() + writeTrailer() + output.close() + } + + private fun writeHeader() { + // stream header + output.writeByte('B'.code) + output.writeByte('Z'.code) + output.writeByte('h'.code) + output.writeByte(('0' + blockSize100k).code) + } + + private fun compressBlock() { + if (blockSize != 0) { + blockSort() + + // block magic + output.writeByte(0x31) + output.writeByte(0x41) + output.writeByte(0x59) + output.writeByte(0x26) + output.writeByte(0x53) + output.writeByte(0x59) + + // block checksum + val blockCrc = blockCrc.value.toInt() + combinedCrc = (combinedCrc shl 1) or (combinedCrc ushr 31) + combinedCrc = combinedCrc xor blockCrc + + output.writeInt(blockCrc) + + // not randomised + output.writeBoolean(false) + + // original pointer + output.writeBits(24, origPtr) + + TODO() // MTF + } + + // reset state ready for next block + blockCrc.reset() + blockSize = 0 + stateInCh = 256 + stateInLen = 0 + inUse.fill(false) + } + + private fun writeTrailer() { + // stream magic + output.writeByte(0x17) + output.writeByte(0x72) + output.writeByte(0x45) + output.writeByte(0x38) + output.writeByte(0x50) + output.writeByte(0x90) + + // stream checksum + output.writeInt(combinedCrc) + } + + private fun blockSort() { + fallbackSort() // TODO: add mainSort() support + + origPtr = -1 + + for (i in 0 until blockSize) { + if (ptr[i] == 0) { + origPtr = i + break + } + } + + check(origPtr != -1) + } + + private fun fallbackSort() { + TODO() + } + + public companion object { + public const val DEFAULT_WORK_FACTOR: Int = 30 + + private const val N_RADIX = 2 + private const val N_QSORT = 12 + private const val N_SHELL = 18 + private const val N_OVERSHOOT = N_RADIX + N_QSORT + N_SHELL + 2 + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index faefb546..11c01ea9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -30,6 +30,7 @@ include( "cache-550", "cli", "compress", + "compress-bzip2", "compress-cli", "conf", "crc32",