From c21895f052347d85a7dd8e94ce5e8566c2c66d33 Mon Sep 17 00:00:00 2001 From: Graham Date: Thu, 31 Mar 2022 20:56:37 +0100 Subject: [PATCH] Add JNR-based bzip2 implementation compatible with Jagex's Signed-off-by: Graham --- compress/build.gradle.kts | 1 + .../org/openrs2/compress/bzip2/Bzip2.kt | 20 +++- .../compress/bzip2/Bzip2OutputStream.kt | 98 +++++++++++++++++++ .../org/openrs2/compress/bzip2/LibBzip2.kt | 64 ++++++++++++ gradle/libs.versions.toml | 1 + 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt create mode 100644 compress/src/main/kotlin/org/openrs2/compress/bzip2/LibBzip2.kt diff --git a/compress/build.gradle.kts b/compress/build.gradle.kts index b788101c..cbaf5651 100644 --- a/compress/build.gradle.kts +++ b/compress/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { api(libs.xz) implementation(projects.util) + implementation(libs.jnr) } publishing { diff --git a/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2.kt b/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2.kt index 6d822266..230cf313 100644 --- a/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2.kt +++ b/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2.kt @@ -1,5 +1,6 @@ package org.openrs2.compress.bzip2 +import com.github.michaelbull.logging.InlineLogger import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream import org.openrs2.util.io.SkipOutputStream @@ -17,11 +18,28 @@ public object Bzip2 { ('0' + BLOCK_SIZE).code.toByte() ) + private val logger = InlineLogger() + private val library: LibBzip2? = try { + LibBzip2.load() + } catch (ex: Throwable) { + logger.warn(ex) { + "Falling back to pure Java bzip2 implementation, " + + "output may not be bit-for-bit identical to Jagex's implementation" + } + + null + } + public fun createHeaderlessInputStream(input: InputStream): InputStream { return BZip2CompressorInputStream(SequenceInputStream(ByteArrayInputStream(HEADER), input)) } public fun createHeaderlessOutputStream(output: OutputStream): OutputStream { - return BZip2CompressorOutputStream(SkipOutputStream(output, HEADER.size.toLong()), BLOCK_SIZE) + val skipOutput = SkipOutputStream(output, HEADER.size.toLong()) + return if (library != null) { + Bzip2OutputStream(library, skipOutput, BLOCK_SIZE) + } else { + BZip2CompressorOutputStream(skipOutput, BLOCK_SIZE) + } } } diff --git a/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt b/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt new file mode 100644 index 00000000..f3b8ea75 --- /dev/null +++ b/compress/src/main/kotlin/org/openrs2/compress/bzip2/Bzip2OutputStream.kt @@ -0,0 +1,98 @@ +package org.openrs2.compress.bzip2 + +import jnr.ffi.Runtime +import java.io.FilterOutputStream +import java.io.IOException +import java.io.OutputStream +import kotlin.math.min + +public class Bzip2OutputStream( + private val library: LibBzip2, + out: OutputStream, + blockSize: Int +) : FilterOutputStream(out) { + private val singleByteBuf = ByteArray(1) + private val runtime = Runtime.getRuntime(library) + private val stream = LibBzip2.BzStream(runtime) + private val nextIn = runtime.memoryManager.allocateDirect(BUFFER_SIZE, false) + private val nextOut = runtime.memoryManager.allocateDirect(BUFFER_SIZE, false) + private val buf = ByteArray(BUFFER_SIZE) + private var closed = false + + init { + val result = library.BZ2_bzCompressInit(stream, blockSize, 0, 0) + if (result != LibBzip2.BZ_OK) { + throw IOException("bzCompressInit failed: $result") + } + + stream.nextIn.set(nextIn) + stream.nextOut.set(nextOut) + } + + override fun write(b: Int) { + singleByteBuf[0] = b.toByte() + write(singleByteBuf, 0, singleByteBuf.size) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + var off = off + var remaining = len + + while (remaining > 0) { + val availIn = min(remaining, BUFFER_SIZE) + nextIn.put(0, b, off, availIn) + stream.nextIn.set(nextIn) + stream.availIn.set(availIn) + + stream.nextOut.set(nextOut) + stream.availOut.set(BUFFER_SIZE) + + val result = library.BZ2_bzCompress(stream, LibBzip2.BZ_RUN) + if (result != LibBzip2.BZ_RUN_OK) { + throw IOException("bzCompress failed: $result") + } + + val read = (availIn - stream.availIn.get()).toInt() + off += read + remaining -= read + + val written = (BUFFER_SIZE - stream.availOut.get()).toInt() + nextOut.get(0, buf, 0, written) + out.write(buf, 0, written) + } + } + + override fun close() { + if (closed) { + return + } + + closed = true + + try { + do { + stream.nextOut.set(nextOut) + stream.availOut.set(BUFFER_SIZE) + + val streamEnd = when (val result = library.BZ2_bzCompress(stream, LibBzip2.BZ_FINISH)) { + LibBzip2.BZ_STREAM_END -> true + LibBzip2.BZ_FINISH_OK -> false + else -> throw IOException("bzCompress failed: $result") + } + + val written = (BUFFER_SIZE - stream.availOut.get()).toInt() + nextOut.get(0, buf, 0, written) + out.write(buf, 0, written) + } while (!streamEnd) + } finally { + val result = library.BZ2_bzCompressEnd(stream) + if (result != LibBzip2.BZ_OK) { + throw IOException("bzCompressEnd failed: $result") + } + } + } + + private companion object { + private const val BUFFER_SIZE = 4096 + } +} diff --git a/compress/src/main/kotlin/org/openrs2/compress/bzip2/LibBzip2.kt b/compress/src/main/kotlin/org/openrs2/compress/bzip2/LibBzip2.kt new file mode 100644 index 00000000..2a137703 --- /dev/null +++ b/compress/src/main/kotlin/org/openrs2/compress/bzip2/LibBzip2.kt @@ -0,0 +1,64 @@ +package org.openrs2.compress.bzip2 + +import jnr.ffi.LibraryLoader +import jnr.ffi.LibraryOption +import jnr.ffi.Runtime +import jnr.ffi.Struct +import jnr.ffi.annotations.Direct + +public interface LibBzip2 { + public class BzStream(runtime: Runtime) : Struct(runtime) { + public val nextIn: Pointer = Pointer() + public val availIn: Unsigned32 = Unsigned32() + public val totalInLo32: Unsigned32 = Unsigned32() + public val totalInHi32: Unsigned32 = Unsigned32() + + public val nextOut: Pointer = Pointer() + public val availOut: Unsigned32 = Unsigned32() + public val totalOutLo32: Unsigned32 = Unsigned32() + public val totalOutHi32: Unsigned32 = Unsigned32() + + public val state: Pointer = Pointer() + + public val alloc: Pointer = Pointer() + public val free: Pointer = Pointer() + public val opaque: Pointer = Pointer() + } + + public fun BZ2_bzCompressInit(@Direct stream: BzStream, blockSize100k: Int, verbosity: Int, workFactor: Int): Int + public fun BZ2_bzCompress(stream: BzStream, action: Int): Int + public fun BZ2_bzCompressEnd(stream: BzStream): Int + + public fun BZ2_bzDecompressInit(@Direct stream: BzStream, blockSize100k: Int, verbosity: Int, small: Int): Int + public fun BZ2_bzDecompress(stream: BzStream): Int + public fun BZ2_bzDecompressEnd(stream: BzStream): Int + + public companion object { + public const val BZ_RUN: Int = 0 + public const val BZ_FLUSH: Int = 1 + public const val BZ_FINISH: Int = 2 + + public const val BZ_OK: Int = 0 + public const val BZ_RUN_OK: Int = 1 + public const val BZ_FLUSH_OK: Int = 2 + public const val BZ_FINISH_OK: Int = 3 + public const val BZ_STREAM_END: Int = 4 + public const val BZ_SEQUENCE_ERROR: Int = -1 + public const val BZ_PARAM_ERROR: Int = -2 + public const val BZ_MEM_ERROR: Int = -3 + public const val BZ_DATA_ERROR: Int = -4 + public const val BZ_DATA_ERROR_MAGIC: Int = -5 + public const val BZ_IO_ERROR: Int = -6 + public const val BZ_UNEXPECTED_EOF: Int = -7 + public const val BZ_OUTBUFF_FULL: Int = -8 + public const val BZ_CONFIG_ERROR: Int = -9 + + public fun load(): LibBzip2 { + return LibraryLoader.loadLibrary( + LibBzip2::class.java, mapOf( + LibraryOption.LoadNow to true, + ), "bz2" + ) + } + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1e7fadee..7dee2e79 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -48,6 +48,7 @@ jdom = { module = "org.jdom:jdom2", version = "2.0.6.1" } jgrapht = { module = "org.jgrapht:jgrapht-core", version = "1.5.1" } jimfs = { module = "com.google.jimfs:jimfs", version.ref = "jimfs" } jquery = { module = "org.webjars:jquery", version = "3.6.0" } +jnr = { module = "com.github.jnr:jnr-ffi", version = "2.2.11" } jsoup = { module = "org.jsoup:jsoup", version = "1.14.3" } junit-api = { module = "org.junit.jupiter:junit-jupiter-api", version = { strictly = "5.8.2" } } junit-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }