Add Cache-Control and ETag headers to the exportGroup endpoint

Signed-off-by: Graham <gpe@openrs2.org>
Add archive API documentation
4 changed files with 314 additions and 1 deletions
--- a/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt
+++ b/archive/src/main/kotlin/org/openrs2/archive/web/CachesController.kt
@ -1,10 +1,15 @@
 package org.openrs2.archive.web
 import io.ktor.application.ApplicationCall
 import io.ktor.http.CacheControl
 import io.ktor.http.ContentDisposition
 import io.ktor.http.ContentType
 import io.ktor.http.HttpHeaders
 import io.ktor.http.HttpStatusCode
 import io.ktor.http.content.CachingOptions
 import io.ktor.http.content.EntityTagVersion
 import io.ktor.http.content.caching
 import io.ktor.http.content.versions
 import io.ktor.response.header
 import io.ktor.response.respond
 import io.ktor.response.respondBytes
@ -21,8 +26,10 @@ import org.openrs2.buffer.use
 import org.openrs2.cache.DiskStoreZipWriter
 import org.openrs2.cache.FlatFileStoreTarWriter
 import org.openrs2.compress.gzip.GzipLevelOutputStream
 import org.openrs2.crypto.whirlpool
 import java.nio.file.attribute.FileTime
 import java.time.Instant
 import java.util.Base64
 import java.util.zip.Deflater
 import java.util.zip.ZipEntry
 import java.util.zip.ZipOutputStream
@ -89,8 +96,20 @@ public class CachesController @Inject constructor(
                return
            }
            val etag = Base64.getEncoder().encodeToString(buf.whirlpool().sliceArray(0 until 16))
            val bytes = ByteBufUtil.getBytes(buf, 0, buf.readableBytes(), false)
-            call.respondBytes(bytes, contentType = ContentType.Application.OctetStream)
+            call.respondBytes(bytes, contentType = ContentType.Application.OctetStream) {
                caching = CachingOptions(
                    cacheControl = CacheControl.MaxAge(
                        maxAgeSeconds = 86400,
                        visibility = CacheControl.Visibility.Public,
                    ),
                )
                versions = listOf(
                    EntityTagVersion(etag, weak = false),
                )
            }
        }
    }
--- a/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt
+++ b/archive/src/main/kotlin/org/openrs2/archive/web/WebServer.kt
@ -5,6 +5,8 @@ import io.ktor.application.ApplicationCall
 import io.ktor.application.call
 import io.ktor.application.install
 import io.ktor.features.CORS
 import io.ktor.features.CachingHeaders
 import io.ktor.features.ConditionalHeaders
 import io.ktor.features.ContentNegotiation
 import io.ktor.features.XForwardedHeaderSupport
 import io.ktor.http.ContentType
@ -36,6 +38,9 @@ public class WebServer @Inject constructor(
 ) {
    public fun start(address: String, port: Int) {
        embeddedServer(CIO, host = address, port = port) {
            install(CachingHeaders)
            install(ConditionalHeaders)
            install(CORS) {
                anyHost()
            }
@ -59,6 +64,7 @@ public class WebServer @Inject constructor(
            routing {
                get("/") { call.respond(ThymeleafContent("index.html", emptyMap())) }
                get("/api") { call.respond(ThymeleafContent("api/index.html", mapOf("active" to "api"))) }
                get("/caches") { cachesController.index(call) }
                get("/caches.json") { cachesController.indexJson(call) }
                get("/caches/{scope}/{id}") { cachesController.show(call) }
--- a/archive/src/main/resources/org/openrs2/archive/templates/api/index.html
+++ b/archive/src/main/resources/org/openrs2/archive/templates/api/index.html
@ -0,0 +1,285 @@
 <!DOCTYPE html>
 <html xmlns:th="http://www.thymeleaf.org" lang="en">
 	<head th:replace="layout.html :: head">
 		<title>API - OpenRS2 Archive</title>
 		<link rel="stylesheet" href="/webjars/bootstrap/css/bootstrap.min.css" />
 		<link rel="stylesheet" href="/static/css/openrs2.css" />
 		<script src="/webjars/jquery/jquery.min.js" defer></script>
 		<script src="/webjars/bootstrap/js/bootstrap.bundle.min.js" defer></script>
 	</head>
 	<body>
 		<nav th:replace="layout.html :: nav"></nav>
 		<main class="container">
 			<h1>API</h1>
 			<p>All endpoints accept requests from any origin. Range requests are not supported by any endpoint.</p>
 			<h2><code>GET /caches.json</code></h2>
 			<p>
 				Returns a list of all caches, including all data available on the main <a href="/caches">caches</a>
 				page, in JSON format:
 			</p>
 			<pre><code>[
    {
        // The cache's internal ID.
        "id": 1,
        // A scope is a group of related games. Missing groups are only located
        // from caches for games in the same scope.
        //
        // Currently the "runescape" scope is used for the "runescape" and
        // "oldschool" games. Each FunOrb game has its own scope.
        //
        // Your code must be prepared for new scopes to be added in the future.
        "scope": "runescape",
        // The game's name. Your code must be prepared for new games to be
        // added in the future.
        "game": "runescape",
        // Currently either "live" or "beta", but your code must be prepared
        // for new environments to be added in the future.
        "environment": "live",
        // The language's ISO-639-1 code. Currently either "en", "de", "fr" or
        // "pt", but your code must be prepared for new languages to be added
        // in the future.
        "language": "en",
        // A list of build numbers the cache is associated with, which may be
        // empty if the build number(s) are not known.
        "builds": [
            {
                // The major number is always set.
                "major": 549,
                // The minor number may be null.
                "minor": null
            },
            {
                "major": 550,
                "minor": null
            }
        ],
        // The earliest timestamp the cache was available to users, in ISO 8601
        // format. May be null if not known.
        "timestamp": "2009-06-12T14:55:58Z",
        // A list of users who provided a copy of this cache.
        //
        // May be empty if the users wished to remain anonymous.
        //
        // The value "Jagex" indicates the cache was directly downloaded from
        // Jagex's servers by the OpenRS2 project, so we are completely certain
        // it is genuine. This value will never be used for a cache obtained
        // from a third party.
        "sources": [
            "Erand",
            "Hlwys",
            "Jagex",
            "K4rn4ge",
            "Nathan",
            "Rune-Wars"
        ],
        // In old engine caches, the number of valid .jag archives that are not
        // missing.
        //
        // In new engine caches, the number of valid JS5 indexes that are not
        // missing.
        //
        // May be null if the cache is still being processed.
        "valid_indexes": 29,
        // In old engine caches, the total number of .jag archives that should
        // exist, based on the cache's CRC table.
        //
        // In new engine caches, the total number of JS5 indexes that should
        // exist, based on the JS5 master index.
        //
        // May be null if the cache is still being processed.
        "indexes": 29,
        // The number of valid files (old engine) or valid groups (new engine)
        // that are not missing. May be null if the cache is still being processed.
        "valid_groups": 71002,
        // In old engine caches, the total number of files that should exist,
        // based on the cache's versionlist.jag archive.
        //
        // In new engine caches, the total number of groups that should exist,
        // based on the JS5 indexes that are available.
        //
        // May be null if the cache is still being processed.
        "groups": 71146,
        // The number of encrypted groups for which a valid key is available.
        // May be null if the cache is still being processed.
        "valid_keys": 1203,
        // The total number of encrypted groups in the cache. May be null if
        // the cache is still being processed.
        "keys": 1240,
        // The total size of all groups in the cache in bytes. May be null if
        // the cache is still being processed.
        "size": 74970573,
        // The number of 520-byte blocks required to store the cache's data in
        // a .dat2 file. May be null if the cache is still being processed.
        "blocks": 185273,
        // A boolean flag indicating if the cache is small enough to be
        // downloaded in .dat2/.idx format. May be null if the cache is still
        // being processed.
        "disk_store_valid": true
    },
    ...
 ]</code></pre>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/disk.zip</code></h2>
 			<p>
 				Returns a cache as a ZIP archive of <code>.dat/.idx</code>
 				(old engine) or <code>.dat2/.idx</code> (new engine) files. All
 				files are stored underneath a <code>cache</code> subdirectory
 				in the zip archive.
 			</p>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/flat-file.tar.gz</code></h2>
 			<p>
 				Returns a cache as a gzipped tarball of files, where each
 				file in the tarball holds a single file from the cache (old
 				engine) or single group (new engine).
 			</p>
 			<p>
 				The paths within the archive all have a format of
 				<code>cache/&lt;index&gt;/&lt;file&gt;.dat</code> (old engine)
 				or <code>cache/&lt;archive&gt;/&lt;group&gt;.dat</code> (new
 				engine).
 			</p>
 			<p>The two byte version trailers are included.</p>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/keys.json</code></h2>
 			<p>Returns a list of valid XTEA keys for the cache in JSON format:</p>
 			<pre><code>[
    {
        // The ID of the archive containing the group the key is used for.
        // Typically this is 5 (maps), but do note that RuneScape 3 does
        // support encrypting interfaces, though the functionality has not yet
        // been used, and some FunOrb games also have encrypted groups.
        "archive": 5,
        // The ID of the group the key is used for.
        "group": 1,
        // The group's name hash, or null if the group has no name.
        "name_hash": -1153472937,
        // The name of the group, if available, or null if the group has no
        // name or if the name is not known.
        "name": "l40_55",
        // The ID of the map square, if the group is an encrypted loc group
        // (has a name of lX_Z). The map square ID is ((X &lt;&lt; 8) | Z).
        // null if the group is not an encrypted loc group.
        "mapsquare": 10295,
        // The XTEA key, represented as four 32-bit integers.
        "key": [
            -1920480496,
            -1423914110,
            951774544,
            -1419269290
        ]
    },
    ...
 ]</code></pre>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/keys.zip</code></h2>
 			<p>
 				Returns a zip archive file of valid XTEA keys for loc groups.
 				Each key is stored in a text file containing four lines, with
 				each line containing a 32-bit component of the key as a decimal
 				string. The paths within the archive all have a format of
 				<code>keys/&lt;mapsquare&gt;.txt</code>.
 			</p>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/map.png</code></h2>
 			<p>
 				Renders the map squares in the cache, with a coloured outline
 				representing whether we have a valid key for each map square or
 				not:
 			</p>
 			<ul>
 				<li><strong>Valid key:</strong> green outline.</li>
 				<li><strong>Loc group is not encrypted:</strong> green outline.</li>
 				<li><strong>Empty loc group:</strong> grey outline.</li>
 				<li><strong>Key unknown:</strong> red outline.</li>
 			</ul>
 			<p>
 				Empty loc groups may be replaced with an unencrypted equivalent
 				with a cache editor.
 			</p>
 			<h2><code>GET /caches/&lt;scope&gt;/&lt;id&gt;/archives/&lt;archive&gt;/groups/&lt;group&gt;.dat</code></h2>
 			<p>
 				Returns a single file (old engine) or group (new engine) in
 				binary format. The response contains a <code>.jag</code>
 				archive (index 0 of an old engine cache), a GZIP-compressed
 				file (the remaining indexes of an old engine cache) or
 				JS5-compressed data (new engine cache, also known as a
 				container). The two byte version trailer is not included.
 			</p>
 			<h2><code>GET /keys/all.json</code></h2>
 			<p>
 				Returns a list of all XTEA keys in the database, including
 				candidate keys that have not been validated against any cache.
 			</p>
 			<pre><code>[
    // The XTEA key, represented as four 32-bit integers.
    [
        -2147135705,
        1113423446,
        1294100345,
        946019601
    ],
    ...
 ]</code></pre>
 			<h2><code>GET /keys/valid.json</code></h2>
 			<p>
 				Returns a list of XTEA keys in the database, only including
 				keys validated against at least one cache.
 			</p>
 			<pre><code>[
    // The XTEA key, represented as four 32-bit integers.
    [
        -2147135705,
        1113423446,
        1294100345,
        946019601
    ],
    ...
 ]</code></pre>
 		</main>
 	</body>
 </html>
--- a/archive/src/main/resources/org/openrs2/archive/templates/layout.html
+++ b/archive/src/main/resources/org/openrs2/archive/templates/layout.html
@ -26,6 +26,9 @@
 					<li class="nav-item">
 						<a class="nav-link" th:classappend="${active == 'keys'}? 'active'" href="/keys">Keys</a>
 					</li>
 					<li class="nav-item">
 						<a class="nav-link" th:classappend="${active == 'api'}? 'active'" href="/api">API</a>
 					</li>
 					<li class="nav-item">
 						<a class="nav-link" href="/pub">Other</a>
 					</li>
Author	SHA1	Message	Date
Graham	e804fdc065	Add Cache-Control and ETag headers to the exportGroup endpoint continuous-integration/drone/push Build is passing Details Signed-off-by: Graham <gpe@openrs2.org>	2 years ago
Graham	9c9a1ecf39	Add archive API documentation	2 years ago