From 36f5efa1ad51a9ffd3a1f0c1f2308b35b89c39e4 Mon Sep 17 00:00:00 2001 From: Graham Date: Wed, 1 Sep 2021 10:47:25 +0100 Subject: [PATCH] Serve flat file caches as .tar.gz files instead of .zip files Streaming .tar.gz files requires less memory, as we don't need to remember metadata about each file for the end of directory record. Signed-off-by: Graham --- archive/build.gradle.kts | 1 + .../openrs2/archive/web/CachesController.kt | 11 ++++-- .../org/openrs2/archive/web/WebServer.kt | 2 +- .../archive/templates/caches/index.html | 2 +- .../archive/templates/caches/show.html | 2 +- cache/build.gradle.kts | 1 + ...ZipWriter.kt => FlatFileStoreTarWriter.kt} | 35 ++++++++---------- ...rTest.kt => FlatFileStoreTarWriterTest.kt} | 16 ++++---- .../cache/flat-file-store-tar/cache.tar | Bin 0 -> 5632 bytes .../cache/flat-file-store-zip/cache.zip | Bin 647 -> 0 bytes 10 files changed, 35 insertions(+), 35 deletions(-) rename cache/src/main/kotlin/org/openrs2/cache/{FlatFileStoreZipWriter.kt => FlatFileStoreTarWriter.kt} (69%) rename cache/src/test/kotlin/org/openrs2/cache/{FlatFileStoreZipWriterTest.kt => FlatFileStoreTarWriterTest.kt} (83%) create mode 100644 cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar delete mode 100644 cache/src/test/resources/org/openrs2/cache/flat-file-store-zip/cache.zip diff --git a/archive/build.gradle.kts b/archive/build.gradle.kts index d4d34a32..a521cda6 100644 --- a/archive/build.gradle.kts +++ b/archive/build.gradle.kts @@ -14,6 +14,7 @@ dependencies { implementation(projects.buffer) implementation(projects.cache550) implementation(projects.cli) + implementation(projects.compress) implementation(projects.db) implementation(projects.http) implementation(projects.inject) diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt index 29746e2e..f0c3db96 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt @@ -12,10 +12,12 @@ import io.ktor.thymeleaf.ThymeleafContent import io.netty.buffer.ByteBufAllocator import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import org.openrs2.archive.cache.CacheExporter import org.openrs2.archive.map.MapRenderer import org.openrs2.cache.DiskStoreZipWriter -import org.openrs2.cache.FlatFileStoreZipWriter +import org.openrs2.cache.FlatFileStoreTarWriter +import org.openrs2.compress.gzip.GzipLevelOutputStream import java.nio.file.attribute.FileTime import java.time.Instant import java.util.zip.Deflater @@ -85,12 +87,13 @@ public class CachesController @Inject constructor( call.response.header( HttpHeaders.ContentDisposition, ContentDisposition.Attachment - .withParameter(ContentDisposition.Parameters.FileName, "cache.zip") + .withParameter(ContentDisposition.Parameters.FileName, "cache.tar.gz") .toString() ) - call.respondOutputStream(contentType = ContentType.Application.Zip) { - FlatFileStoreZipWriter(ZipOutputStream(this)).use { store -> + call.respondOutputStream(contentType = ContentType.Application.GZip) { + val output = TarArchiveOutputStream(GzipLevelOutputStream(this, Deflater.BEST_COMPRESSION)) + FlatFileStoreTarWriter(output).use { store -> exporter.export(id, store) } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt index 271b2e60..80a70f10 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt @@ -78,7 +78,7 @@ public class WebServer @Inject constructor( } } get("/caches/{id}/disk.zip") { cachesController.exportDisk(call) } - get("/caches/{id}/flat-file.zip") { cachesController.exportFlatFile(call) } + get("/caches/{id}/flat-file.tar.gz") { cachesController.exportFlatFile(call) } get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) } get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) } get("/caches/{id}/map.png") { cachesController.renderMap(call) } diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index 4208940c..b7e1f1c1 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -82,7 +82,7 @@
  • Cache (.dat2/.idx)
    • -
  • Cache (Flat file)
  • diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html index e3eba509..cc7a5ee8 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html @@ -54,7 +54,7 @@ Cache (.dat2/.idx) - Cache (Flat file)
    diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 0a75fa74..4d10c98a 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { api(projects.crypto) + api(libs.commons.compress) api(libs.fastutil) api(libs.guice) api(libs.netty.buffer) diff --git a/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt b/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt similarity index 69% rename from cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt rename to cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt index 3a1b6125..3431ac16 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt @@ -1,15 +1,14 @@ package org.openrs2.cache import io.netty.buffer.ByteBuf -import java.nio.file.attribute.FileTime +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import java.time.Instant -import java.util.zip.Deflater -import java.util.zip.ZipEntry -import java.util.zip.ZipOutputStream +import java.util.Date /** * A specialised [Store] implementation that writes a cache in the - * [FlatFileStore] format to a [ZipOutputStream]. + * [FlatFileStore] format to a [TarArchiveOutputStream]. * * The cache is not buffered to disk. * @@ -18,23 +17,17 @@ import java.util.zip.ZipOutputStream * * It is only intended for use by the cache archiving service's web interface. */ -public class FlatFileStoreZipWriter( - private val out: ZipOutputStream, +public class FlatFileStoreTarWriter( + private val out: TarArchiveOutputStream, private val prefix: String = "cache/", - level: Int = Deflater.BEST_COMPRESSION, timestamp: Instant = Instant.EPOCH ) : Store { - private val timestamp = FileTime.from(timestamp) + private val timestamp = Date.from(timestamp) - init { - out.setLevel(level) - } - - private fun createZipEntry(name: String): ZipEntry { - val entry = ZipEntry(prefix + name) - entry.creationTime = timestamp - entry.lastAccessTime = timestamp - entry.lastModifiedTime = timestamp + private fun createTarEntry(name: String, size: Int): TarArchiveEntry { + val entry = TarArchiveEntry(prefix + name) + entry.modTime = timestamp + entry.size = size.toLong() return entry } @@ -57,7 +50,8 @@ public class FlatFileStoreZipWriter( override fun create(archive: Int) { require(archive in 0..Store.MAX_ARCHIVE) - out.putNextEntry(createZipEntry("$archive/")) + out.putArchiveEntry(createTarEntry("$archive/", size = 0)) + out.closeArchiveEntry() } override fun read(archive: Int, group: Int): ByteBuf { @@ -69,8 +63,9 @@ public class FlatFileStoreZipWriter( require(group >= 0) require(buf.readableBytes() <= Store.MAX_GROUP_SIZE) - out.putNextEntry(createZipEntry("$archive/$group.dat")) + out.putArchiveEntry(createTarEntry("$archive/$group.dat", buf.readableBytes())) buf.readBytes(out, buf.readableBytes()) + out.closeArchiveEntry() } override fun remove(archive: Int) { diff --git a/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt b/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt similarity index 83% rename from cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt rename to cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt index 4e371c90..c7194f20 100644 --- a/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt +++ b/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt @@ -3,21 +3,21 @@ package org.openrs2.cache import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs import io.netty.buffer.Unpooled +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import org.openrs2.buffer.copiedBuffer import org.openrs2.buffer.use import org.openrs2.util.io.recursiveEquals import java.io.OutputStream import java.nio.file.Files import java.nio.file.Path -import java.util.zip.ZipOutputStream import kotlin.test.Test import kotlin.test.assertFailsWith import kotlin.test.assertTrue -public class FlatFileStoreZipWriterTest { +public class FlatFileStoreTarWriterTest { @Test fun testBounds() { - FlatFileStoreZipWriter(ZipOutputStream(OutputStream.nullOutputStream())).use { store -> + FlatFileStoreTarWriter(TarArchiveOutputStream(OutputStream.nullOutputStream())).use { store -> // create assertFailsWith { store.create(-1) @@ -58,7 +58,7 @@ public class FlatFileStoreZipWriterTest { @Test fun testUnsupported() { - FlatFileStoreZipWriter(ZipOutputStream(OutputStream.nullOutputStream())).use { store -> + FlatFileStoreTarWriter(TarArchiveOutputStream(OutputStream.nullOutputStream())).use { store -> assertFailsWith { store.exists(0) } @@ -92,11 +92,11 @@ public class FlatFileStoreZipWriterTest { @Test fun testWrite() { Jimfs.newFileSystem(Configuration.forCurrentPlatform()).use { fs -> - val actual = fs.rootDirectories.first().resolve("zip") + val actual = fs.rootDirectories.first().resolve("tar") Files.createDirectories(actual) - Files.newOutputStream(actual.resolve("cache.zip")).use { out -> - FlatFileStoreZipWriter(ZipOutputStream(out)).use { store -> + Files.newOutputStream(actual.resolve("cache.tar")).use { out -> + FlatFileStoreTarWriter(TarArchiveOutputStream(out)).use { store -> store.create(0) copiedBuffer("OpenRS2").use { buf -> @@ -119,7 +119,7 @@ public class FlatFileStoreZipWriterTest { private companion object { private val ROOT = Path.of( - FlatFileStoreZipWriterTest::class.java.getResource("flat-file-store-zip").toURI() + FlatFileStoreTarWriterTest::class.java.getResource("flat-file-store-tar").toURI() ) } } diff --git a/cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar b/cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar new file mode 100644 index 0000000000000000000000000000000000000000..f972b2b5350bb8f062e51cc3a862f045c70f6844 GIT binary patch literal 5632 zcmeHJTMoh?5acMFK$iD0O!NcW^XUPUwc7~bYoj(r@7^ye_ z6j-e(#4(SrY|h0w56dN?y5gP6(U1G+ZUmqtR!(AXr4hRFnAY3;n}5MzcXDH;;=lon zQkmm<49MA>OZ;;a2%%qdvZS9s=YM}4E{CI7f)uXl5P__JqqWqPJm7==xv<3>lKusk z7WLoaxh4H$5D3B6AZVeF-fK+#-~5pOEBya8d3xx2TXQ{REHl)LVg%q|5M)SBOwLHvH_#6a;bmY)NHH)lGX!|EgY+;kL4^Zw z8pI1X$VlHnFD0=A!?e%(9%s3^%$UH2zUGRb!3i`JWFKxr1(6IjGc`3f#b@v*`K4|v z7rHTm%|BrztSA69dk@g%#G7wMrumFaA`GbE3kxs~29Ph20V1?nA)y`MjjA0vXn0Yy zdjidYXorU}4l|IwFNk8sJR~!af|O)){-K!z4LZ0vtZX1pvjCwmBLhPm5Hm0U07qd@ Ao&W#<