From 36f5efa1ad51a9ffd3a1f0c1f2308b35b89c39e4 Mon Sep 17 00:00:00 2001 From: Graham Date: Wed, 1 Sep 2021 10:47:25 +0100 Subject: [PATCH] Serve flat file caches as .tar.gz files instead of .zip files Streaming .tar.gz files requires less memory, as we don't need to remember metadata about each file for the end of directory record. Signed-off-by: Graham --- archive/build.gradle.kts | 1 + .../openrs2/archive/web/CachesController.kt | 11 ++++-- .../org/openrs2/archive/web/WebServer.kt | 2 +- .../archive/templates/caches/index.html | 2 +- .../archive/templates/caches/show.html | 2 +- cache/build.gradle.kts | 1 + ...ZipWriter.kt => FlatFileStoreTarWriter.kt} | 35 ++++++++---------- ...rTest.kt => FlatFileStoreTarWriterTest.kt} | 16 ++++---- .../cache/flat-file-store-tar/cache.tar | Bin 0 -> 5632 bytes .../cache/flat-file-store-zip/cache.zip | Bin 647 -> 0 bytes 10 files changed, 35 insertions(+), 35 deletions(-) rename cache/src/main/kotlin/org/openrs2/cache/{FlatFileStoreZipWriter.kt => FlatFileStoreTarWriter.kt} (69%) rename cache/src/test/kotlin/org/openrs2/cache/{FlatFileStoreZipWriterTest.kt => FlatFileStoreTarWriterTest.kt} (83%) create mode 100644 cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar delete mode 100644 cache/src/test/resources/org/openrs2/cache/flat-file-store-zip/cache.zip diff --git a/archive/build.gradle.kts b/archive/build.gradle.kts index d4d34a3288..a521cda6be 100644 --- a/archive/build.gradle.kts +++ b/archive/build.gradle.kts @@ -14,6 +14,7 @@ dependencies { implementation(projects.buffer) implementation(projects.cache550) implementation(projects.cli) + implementation(projects.compress) implementation(projects.db) implementation(projects.http) implementation(projects.inject) diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt index 29746e2ec8..f0c3db96dc 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt @@ -12,10 +12,12 @@ import io.ktor.thymeleaf.ThymeleafContent import io.netty.buffer.ByteBufAllocator import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import org.openrs2.archive.cache.CacheExporter import org.openrs2.archive.map.MapRenderer import org.openrs2.cache.DiskStoreZipWriter -import org.openrs2.cache.FlatFileStoreZipWriter +import org.openrs2.cache.FlatFileStoreTarWriter +import org.openrs2.compress.gzip.GzipLevelOutputStream import java.nio.file.attribute.FileTime import java.time.Instant import java.util.zip.Deflater @@ -85,12 +87,13 @@ public class CachesController @Inject constructor( call.response.header( HttpHeaders.ContentDisposition, ContentDisposition.Attachment - .withParameter(ContentDisposition.Parameters.FileName, "cache.zip") + .withParameter(ContentDisposition.Parameters.FileName, "cache.tar.gz") .toString() ) - call.respondOutputStream(contentType = ContentType.Application.Zip) { - FlatFileStoreZipWriter(ZipOutputStream(this)).use { store -> + call.respondOutputStream(contentType = ContentType.Application.GZip) { + val output = TarArchiveOutputStream(GzipLevelOutputStream(this, Deflater.BEST_COMPRESSION)) + FlatFileStoreTarWriter(output).use { store -> exporter.export(id, store) } } diff --git a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt index 271b2e6058..80a70f10bc 100644 --- a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt +++ b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt @@ -78,7 +78,7 @@ public class WebServer @Inject constructor( } } get("/caches/{id}/disk.zip") { cachesController.exportDisk(call) } - get("/caches/{id}/flat-file.zip") { cachesController.exportFlatFile(call) } + get("/caches/{id}/flat-file.tar.gz") { cachesController.exportFlatFile(call) } get("/caches/{id}/keys.json") { cachesController.exportKeysJson(call) } get("/caches/{id}/keys.zip") { cachesController.exportKeysZip(call) } get("/caches/{id}/map.png") { cachesController.renderMap(call) } diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html index 4208940c43..b7e1f1c1ad 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/index.html @@ -82,7 +82,7 @@
  • Cache (.dat2/.idx)
    • -
  • Cache (Flat file)
  • diff --git a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html index e3eba5092f..cc7a5ee8ed 100644 --- a/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html +++ b/archive/src/main/resources/org/openrs2/archive/templates/caches/show.html @@ -54,7 +54,7 @@ Cache (.dat2/.idx) - Cache (Flat file)
    diff --git a/cache/build.gradle.kts b/cache/build.gradle.kts index 0a75fa74c8..4d10c98a93 100644 --- a/cache/build.gradle.kts +++ b/cache/build.gradle.kts @@ -5,6 +5,7 @@ plugins { dependencies { api(projects.crypto) + api(libs.commons.compress) api(libs.fastutil) api(libs.guice) api(libs.netty.buffer) diff --git a/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt b/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt similarity index 69% rename from cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt rename to cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt index 3a1b612505..3431ac1613 100644 --- a/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreZipWriter.kt +++ b/cache/src/main/kotlin/org/openrs2/cache/FlatFileStoreTarWriter.kt @@ -1,15 +1,14 @@ package org.openrs2.cache import io.netty.buffer.ByteBuf -import java.nio.file.attribute.FileTime +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import java.time.Instant -import java.util.zip.Deflater -import java.util.zip.ZipEntry -import java.util.zip.ZipOutputStream +import java.util.Date /** * A specialised [Store] implementation that writes a cache in the - * [FlatFileStore] format to a [ZipOutputStream]. + * [FlatFileStore] format to a [TarArchiveOutputStream]. * * The cache is not buffered to disk. * @@ -18,23 +17,17 @@ import java.util.zip.ZipOutputStream * * It is only intended for use by the cache archiving service's web interface. */ -public class FlatFileStoreZipWriter( - private val out: ZipOutputStream, +public class FlatFileStoreTarWriter( + private val out: TarArchiveOutputStream, private val prefix: String = "cache/", - level: Int = Deflater.BEST_COMPRESSION, timestamp: Instant = Instant.EPOCH ) : Store { - private val timestamp = FileTime.from(timestamp) + private val timestamp = Date.from(timestamp) - init { - out.setLevel(level) - } - - private fun createZipEntry(name: String): ZipEntry { - val entry = ZipEntry(prefix + name) - entry.creationTime = timestamp - entry.lastAccessTime = timestamp - entry.lastModifiedTime = timestamp + private fun createTarEntry(name: String, size: Int): TarArchiveEntry { + val entry = TarArchiveEntry(prefix + name) + entry.modTime = timestamp + entry.size = size.toLong() return entry } @@ -57,7 +50,8 @@ public class FlatFileStoreZipWriter( override fun create(archive: Int) { require(archive in 0..Store.MAX_ARCHIVE) - out.putNextEntry(createZipEntry("$archive/")) + out.putArchiveEntry(createTarEntry("$archive/", size = 0)) + out.closeArchiveEntry() } override fun read(archive: Int, group: Int): ByteBuf { @@ -69,8 +63,9 @@ public class FlatFileStoreZipWriter( require(group >= 0) require(buf.readableBytes() <= Store.MAX_GROUP_SIZE) - out.putNextEntry(createZipEntry("$archive/$group.dat")) + out.putArchiveEntry(createTarEntry("$archive/$group.dat", buf.readableBytes())) buf.readBytes(out, buf.readableBytes()) + out.closeArchiveEntry() } override fun remove(archive: Int) { diff --git a/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt b/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt similarity index 83% rename from cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt rename to cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt index 4e371c90e7..c7194f20b0 100644 --- a/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreZipWriterTest.kt +++ b/cache/src/test/kotlin/org/openrs2/cache/FlatFileStoreTarWriterTest.kt @@ -3,21 +3,21 @@ package org.openrs2.cache import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs import io.netty.buffer.Unpooled +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream import org.openrs2.buffer.copiedBuffer import org.openrs2.buffer.use import org.openrs2.util.io.recursiveEquals import java.io.OutputStream import java.nio.file.Files import java.nio.file.Path -import java.util.zip.ZipOutputStream import kotlin.test.Test import kotlin.test.assertFailsWith import kotlin.test.assertTrue -public class FlatFileStoreZipWriterTest { +public class FlatFileStoreTarWriterTest { @Test fun testBounds() { - FlatFileStoreZipWriter(ZipOutputStream(OutputStream.nullOutputStream())).use { store -> + FlatFileStoreTarWriter(TarArchiveOutputStream(OutputStream.nullOutputStream())).use { store -> // create assertFailsWith { store.create(-1) @@ -58,7 +58,7 @@ public class FlatFileStoreZipWriterTest { @Test fun testUnsupported() { - FlatFileStoreZipWriter(ZipOutputStream(OutputStream.nullOutputStream())).use { store -> + FlatFileStoreTarWriter(TarArchiveOutputStream(OutputStream.nullOutputStream())).use { store -> assertFailsWith { store.exists(0) } @@ -92,11 +92,11 @@ public class FlatFileStoreZipWriterTest { @Test fun testWrite() { Jimfs.newFileSystem(Configuration.forCurrentPlatform()).use { fs -> - val actual = fs.rootDirectories.first().resolve("zip") + val actual = fs.rootDirectories.first().resolve("tar") Files.createDirectories(actual) - Files.newOutputStream(actual.resolve("cache.zip")).use { out -> - FlatFileStoreZipWriter(ZipOutputStream(out)).use { store -> + Files.newOutputStream(actual.resolve("cache.tar")).use { out -> + FlatFileStoreTarWriter(TarArchiveOutputStream(out)).use { store -> store.create(0) copiedBuffer("OpenRS2").use { buf -> @@ -119,7 +119,7 @@ public class FlatFileStoreZipWriterTest { private companion object { private val ROOT = Path.of( - FlatFileStoreZipWriterTest::class.java.getResource("flat-file-store-zip").toURI() + FlatFileStoreTarWriterTest::class.java.getResource("flat-file-store-tar").toURI() ) } } diff --git a/cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar b/cache/src/test/resources/org/openrs2/cache/flat-file-store-tar/cache.tar new file mode 100644 index 0000000000000000000000000000000000000000..f972b2b5350bb8f062e51cc3a862f045c70f6844 GIT binary patch literal 5632 zcmeHJTMoh?5acMFK$iD0O!NcW^XUPUwc7~bYoj(r@7^ye_ z6j-e(#4(SrY|h0w56dN?y5gP6(U1G+ZUmqtR!(AXr4hRFnAY3;n}5MzcXDH;;=lon zQkmm<49MA>OZ;;a2%%qdvZS9s=YM}4E{CI7f)uXl5P__JqqWqPJm7==xv<3>lKusk z7WLoaxh4H$5D3B6AZVeF-fK+#-~5pOEBya8d3xx2TXQ{REHl)LVg%q|5M)SBOwLHvH_#6a;bmY)NHH)lGX!|EgY+;kL4^Zw z8pI1X$VlHnFD0=A!?e%(9%s3^%$UH2zUGRb!3i`JWFKxr1(6IjGc`3f#b@v*`K4|v z7rHTm%|BrztSA69dk@g%#G7wMrumFaA`GbE3kxs~29Ph20V1?nA)y`MjjA0vXn0Yy zdjidYXorU}4l|IwFNk8sJR~!af|O)){-K!z4LZ0vtZX1pvjCwmBLhPm5Hm0U07qd@ Ao&W#<