forked from openrs2/openrs2
Signed-off-by: Graham <gpe@openrs2.org>bzip2
parent
72e9107900
commit
f87d89fe7c
@ -0,0 +1,28 @@ |
|||||||
|
package org.openrs2.archive.cache |
||||||
|
|
||||||
|
import dev.openrs2.net.BootstrapFactory |
||||||
|
import dev.openrs2.net.suspend |
||||||
|
import javax.inject.Inject |
||||||
|
import javax.inject.Singleton |
||||||
|
import kotlin.coroutines.suspendCoroutine |
||||||
|
|
||||||
|
@Singleton |
||||||
|
public class CacheDownloader @Inject constructor( |
||||||
|
private val bootstrapFactory: BootstrapFactory, |
||||||
|
private val importer: CacheImporter |
||||||
|
) { |
||||||
|
public suspend fun download(hostname: String, port: Int, version: Int) { |
||||||
|
val group = bootstrapFactory.createEventLoopGroup() |
||||||
|
try { |
||||||
|
suspendCoroutine<Unit> { continuation -> |
||||||
|
val bootstrap = bootstrapFactory.createBootstrap(group) |
||||||
|
val handler = Js5ChannelHandler(bootstrap, hostname, port, version, continuation, importer) |
||||||
|
|
||||||
|
bootstrap.handler(Js5ChannelInitializer(handler)) |
||||||
|
.connect(hostname, port) |
||||||
|
} |
||||||
|
} finally { |
||||||
|
group.shutdownGracefully().suspend() |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,15 @@ |
|||||||
|
package org.openrs2.archive.cache |
||||||
|
|
||||||
|
import com.github.ajalt.clikt.core.CliktCommand |
||||||
|
import com.google.inject.Guice |
||||||
|
import kotlinx.coroutines.runBlocking |
||||||
|
import org.openrs2.archive.ArchiveModule |
||||||
|
|
||||||
|
public class DownloadCommand : CliktCommand(name = "download") { |
||||||
|
override fun run(): Unit = runBlocking { |
||||||
|
val injector = Guice.createInjector(ArchiveModule) |
||||||
|
val downloader = injector.getInstance(CacheDownloader::class.java) |
||||||
|
// TODO(gpe): make these configurable and/or fetch from the database |
||||||
|
downloader.download("oldschool1.runescape.com", 43594, 193) |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,176 @@ |
|||||||
|
package org.openrs2.archive.cache |
||||||
|
|
||||||
|
import io.netty.bootstrap.Bootstrap |
||||||
|
import io.netty.buffer.ByteBuf |
||||||
|
import io.netty.channel.ChannelHandler |
||||||
|
import io.netty.channel.ChannelHandlerContext |
||||||
|
import io.netty.channel.SimpleChannelInboundHandler |
||||||
|
import kotlinx.coroutines.runBlocking |
||||||
|
import org.openrs2.buffer.use |
||||||
|
import org.openrs2.cache.Js5Archive |
||||||
|
import org.openrs2.cache.Js5Compression |
||||||
|
import org.openrs2.cache.Js5Index |
||||||
|
import org.openrs2.cache.Js5MasterIndex |
||||||
|
import org.openrs2.protocol.Rs2Decoder |
||||||
|
import org.openrs2.protocol.Rs2Encoder |
||||||
|
import org.openrs2.protocol.js5.Js5Request |
||||||
|
import org.openrs2.protocol.js5.Js5RequestEncoder |
||||||
|
import org.openrs2.protocol.js5.Js5Response |
||||||
|
import org.openrs2.protocol.js5.Js5ResponseDecoder |
||||||
|
import org.openrs2.protocol.js5.XorDecoder |
||||||
|
import org.openrs2.protocol.login.LoginRequest |
||||||
|
import org.openrs2.protocol.login.LoginResponse |
||||||
|
import kotlin.coroutines.Continuation |
||||||
|
import kotlin.coroutines.resume |
||||||
|
import kotlin.coroutines.resumeWithException |
||||||
|
|
||||||
|
@ChannelHandler.Sharable |
||||||
|
public class Js5ChannelHandler( |
||||||
|
private val bootstrap: Bootstrap, |
||||||
|
private val hostname: String, |
||||||
|
private val port: Int, |
||||||
|
private var version: Int, |
||||||
|
private val continuation: Continuation<Unit>, |
||||||
|
private val importer: CacheImporter, |
||||||
|
private val maxInFlightRequests: Int = 200, |
||||||
|
maxVersionAttempts: Int = 10 |
||||||
|
) : SimpleChannelInboundHandler<Any>(Object::class.java) { |
||||||
|
private val maxVersion = version + maxVersionAttempts |
||||||
|
private val inFlightRequests = mutableSetOf<Js5Request.Group>() |
||||||
|
private val pendingRequests = ArrayDeque<Js5Request.Group>() |
||||||
|
private lateinit var indexes: Array<Js5Index?> |
||||||
|
private val groups = mutableListOf<CacheImporter.Group>() |
||||||
|
|
||||||
|
override fun channelActive(ctx: ChannelHandlerContext) { |
||||||
|
ctx.writeAndFlush(LoginRequest.InitJs5RemoteConnection(version), ctx.voidPromise()) |
||||||
|
ctx.read() |
||||||
|
} |
||||||
|
|
||||||
|
override fun channelRead0(ctx: ChannelHandlerContext, msg: Any) { |
||||||
|
when (msg) { |
||||||
|
is LoginResponse.Js5Ok -> handleOk(ctx) |
||||||
|
is LoginResponse.ClientOutOfDate -> handleClientOutOfDate(ctx) |
||||||
|
is LoginResponse -> throw Exception("Invalid response: $msg") |
||||||
|
is Js5Response -> handleResponse(ctx, msg) |
||||||
|
else -> throw Exception("Unknown message type: ${msg.javaClass.name}") |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
override fun exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable) { |
||||||
|
releaseGroups() |
||||||
|
ctx.close() |
||||||
|
continuation.resumeWithException(cause) |
||||||
|
} |
||||||
|
|
||||||
|
private fun handleOk(ctx: ChannelHandlerContext) { |
||||||
|
val pipeline = ctx.pipeline() |
||||||
|
|
||||||
|
pipeline.remove(Rs2Encoder::class.java) |
||||||
|
pipeline.remove(Rs2Decoder::class.java) |
||||||
|
pipeline.addFirst( |
||||||
|
Js5RequestEncoder, |
||||||
|
XorDecoder(), |
||||||
|
Js5ResponseDecoder() |
||||||
|
) |
||||||
|
|
||||||
|
val request = Js5Request.Group(false, Js5Archive.ARCHIVESET, Js5Archive.ARCHIVESET) |
||||||
|
pendingRequests += request |
||||||
|
flushRequests(ctx) |
||||||
|
} |
||||||
|
|
||||||
|
private fun handleClientOutOfDate(ctx: ChannelHandlerContext) { |
||||||
|
if (++version > maxVersion) { |
||||||
|
throw Exception("Failed to identify current version") |
||||||
|
} |
||||||
|
|
||||||
|
ctx.close() |
||||||
|
bootstrap.connect(hostname, port) |
||||||
|
} |
||||||
|
|
||||||
|
private fun handleResponse(ctx: ChannelHandlerContext, response: Js5Response) { |
||||||
|
val request = Js5Request.Group(response.prefetch, response.archive, response.group) |
||||||
|
|
||||||
|
val removed = inFlightRequests.remove(request) |
||||||
|
if (!removed) { |
||||||
|
throw Exception("Received response for request not in-flight") |
||||||
|
} |
||||||
|
|
||||||
|
if (response.archive == Js5Archive.ARCHIVESET && response.group == Js5Archive.ARCHIVESET) { |
||||||
|
processMasterIndex(response.data) |
||||||
|
} else if (response.archive == Js5Archive.ARCHIVESET) { |
||||||
|
processIndex(response.group, response.data) |
||||||
|
} else { |
||||||
|
val version = indexes[response.archive]!![response.group]!!.version |
||||||
|
val encrypted = Js5Compression.isEncrypted(response.data.slice()) |
||||||
|
groups += CacheImporter.Group(response.archive, response.group, response.data.retain(), version, encrypted) |
||||||
|
} |
||||||
|
|
||||||
|
val complete = pendingRequests.isEmpty() && inFlightRequests.isEmpty() |
||||||
|
|
||||||
|
if (groups.size >= CacheImporter.BATCH_SIZE || complete) { |
||||||
|
runBlocking { |
||||||
|
importer.importGroups(groups) |
||||||
|
} |
||||||
|
|
||||||
|
releaseGroups() |
||||||
|
} |
||||||
|
|
||||||
|
if (complete) { |
||||||
|
ctx.close() |
||||||
|
continuation.resume(Unit) |
||||||
|
} else { |
||||||
|
flushRequests(ctx) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun processMasterIndex(buf: ByteBuf) { |
||||||
|
val masterIndex = Js5Compression.uncompress(buf.slice()).use { uncompressed -> |
||||||
|
Js5MasterIndex.read(uncompressed) |
||||||
|
} |
||||||
|
|
||||||
|
val rawIndexes = runBlocking { importer.importMasterIndexAndGetIndexes(masterIndex, buf) } |
||||||
|
try { |
||||||
|
indexes = arrayOfNulls(rawIndexes.size) |
||||||
|
|
||||||
|
for ((archive, index) in rawIndexes.withIndex()) { |
||||||
|
if (index != null) { |
||||||
|
processIndex(archive, index) |
||||||
|
} else { |
||||||
|
pendingRequests += Js5Request.Group(false, Js5Archive.ARCHIVESET, archive) |
||||||
|
} |
||||||
|
} |
||||||
|
} finally { |
||||||
|
rawIndexes.filterNotNull().forEach(ByteBuf::release) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun processIndex(archive: Int, buf: ByteBuf) { |
||||||
|
val index = Js5Compression.uncompress(buf.slice()).use { uncompressed -> |
||||||
|
Js5Index.read(uncompressed) |
||||||
|
} |
||||||
|
indexes[archive] = index |
||||||
|
|
||||||
|
val groups = runBlocking { |
||||||
|
importer.importIndexAndGetMissingGroups(archive, index, buf) |
||||||
|
} |
||||||
|
for (group in groups) { |
||||||
|
pendingRequests += Js5Request.Group(false, archive, group) |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
private fun flushRequests(ctx: ChannelHandlerContext) { |
||||||
|
while (inFlightRequests.size < maxInFlightRequests) { |
||||||
|
val request = pendingRequests.removeFirstOrNull() ?: break |
||||||
|
inFlightRequests += request |
||||||
|
ctx.write(request, ctx.voidPromise()) |
||||||
|
} |
||||||
|
|
||||||
|
ctx.flush() |
||||||
|
ctx.read() |
||||||
|
} |
||||||
|
|
||||||
|
private fun releaseGroups() { |
||||||
|
groups.forEach(CacheImporter.Group::release) |
||||||
|
groups.clear() |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,22 @@ |
|||||||
|
package org.openrs2.archive.cache |
||||||
|
|
||||||
|
import io.netty.channel.Channel |
||||||
|
import io.netty.channel.ChannelInitializer |
||||||
|
import org.openrs2.protocol.Protocol |
||||||
|
import org.openrs2.protocol.Rs2Decoder |
||||||
|
import org.openrs2.protocol.Rs2Encoder |
||||||
|
import org.openrs2.protocol.login.ClientOutOfDateCodec |
||||||
|
import org.openrs2.protocol.login.InitJs5RemoteConnectionCodec |
||||||
|
import org.openrs2.protocol.login.IpLimitCodec |
||||||
|
import org.openrs2.protocol.login.Js5OkCodec |
||||||
|
import org.openrs2.protocol.login.ServerFullCodec |
||||||
|
|
||||||
|
public class Js5ChannelInitializer(private val handler: Js5ChannelHandler) : ChannelInitializer<Channel>() { |
||||||
|
override fun initChannel(ch: Channel) { |
||||||
|
ch.pipeline().addLast( |
||||||
|
Rs2Encoder(Protocol(InitJs5RemoteConnectionCodec)), |
||||||
|
Rs2Decoder(Protocol(Js5OkCodec, ClientOutOfDateCodec, IpLimitCodec, ServerFullCodec)), |
||||||
|
handler |
||||||
|
) |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,32 @@ |
|||||||
|
plugins { |
||||||
|
`maven-publish` |
||||||
|
kotlin("jvm") |
||||||
|
} |
||||||
|
|
||||||
|
dependencies { |
||||||
|
api("com.google.inject:guice:${Versions.guice}") |
||||||
|
api("io.netty:netty-transport:${Versions.netty}") |
||||||
|
api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}") |
||||||
|
|
||||||
|
implementation(project(":buffer")) |
||||||
|
implementation("io.netty:netty-transport-native-epoll:${Versions.netty}:linux-aarch_64") |
||||||
|
implementation("io.netty:netty-transport-native-epoll:${Versions.netty}:linux-x86_64") |
||||||
|
implementation("io.netty:netty-transport-native-kqueue:${Versions.netty}:osx-x86_64") |
||||||
|
implementation("io.netty.incubator:netty-incubator-transport-native-io_uring:${Versions.nettyIoUring}:linux-x86_64") |
||||||
|
} |
||||||
|
|
||||||
|
publishing { |
||||||
|
publications.create<MavenPublication>("maven") { |
||||||
|
from(components["java"]) |
||||||
|
|
||||||
|
pom { |
||||||
|
packaging = "jar" |
||||||
|
name.set("OpenRS2 Network") |
||||||
|
description.set( |
||||||
|
""" |
||||||
|
Common Netty utility code. |
||||||
|
""".trimIndent() |
||||||
|
) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,74 @@ |
|||||||
|
package dev.openrs2.net |
||||||
|
|
||||||
|
import io.netty.bootstrap.Bootstrap |
||||||
|
import io.netty.bootstrap.ServerBootstrap |
||||||
|
import io.netty.buffer.ByteBufAllocator |
||||||
|
import io.netty.channel.ChannelOption |
||||||
|
import io.netty.channel.EventLoopGroup |
||||||
|
import io.netty.channel.epoll.Epoll |
||||||
|
import io.netty.channel.epoll.EpollEventLoopGroup |
||||||
|
import io.netty.channel.epoll.EpollServerSocketChannel |
||||||
|
import io.netty.channel.epoll.EpollSocketChannel |
||||||
|
import io.netty.channel.kqueue.KQueue |
||||||
|
import io.netty.channel.kqueue.KQueueEventLoopGroup |
||||||
|
import io.netty.channel.kqueue.KQueueServerSocketChannel |
||||||
|
import io.netty.channel.kqueue.KQueueSocketChannel |
||||||
|
import io.netty.channel.nio.NioEventLoopGroup |
||||||
|
import io.netty.channel.socket.nio.NioServerSocketChannel |
||||||
|
import io.netty.channel.socket.nio.NioSocketChannel |
||||||
|
import io.netty.incubator.channel.uring.IOUring |
||||||
|
import io.netty.incubator.channel.uring.IOUringEventLoopGroup |
||||||
|
import io.netty.incubator.channel.uring.IOUringServerSocketChannel |
||||||
|
import io.netty.incubator.channel.uring.IOUringSocketChannel |
||||||
|
import javax.inject.Inject |
||||||
|
import javax.inject.Singleton |
||||||
|
|
||||||
|
@Singleton |
||||||
|
public class BootstrapFactory @Inject constructor( |
||||||
|
private val alloc: ByteBufAllocator |
||||||
|
) { |
||||||
|
public fun createEventLoopGroup(): EventLoopGroup { |
||||||
|
return when { |
||||||
|
IOUring.isAvailable() -> IOUringEventLoopGroup() |
||||||
|
Epoll.isAvailable() -> EpollEventLoopGroup() |
||||||
|
KQueue.isAvailable() -> KQueueEventLoopGroup() |
||||||
|
else -> NioEventLoopGroup() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public fun createBootstrap(group: EventLoopGroup): Bootstrap { |
||||||
|
val channel = when (group) { |
||||||
|
is IOUringEventLoopGroup -> IOUringSocketChannel::class.java |
||||||
|
is EpollEventLoopGroup -> EpollSocketChannel::class.java |
||||||
|
is KQueueEventLoopGroup -> KQueueSocketChannel::class.java |
||||||
|
is NioEventLoopGroup -> NioSocketChannel::class.java |
||||||
|
else -> throw IllegalArgumentException("Unknown EventLoopGroup type") |
||||||
|
} |
||||||
|
|
||||||
|
return Bootstrap() |
||||||
|
.group(group) |
||||||
|
.channel(channel) |
||||||
|
.option(ChannelOption.ALLOCATOR, alloc) |
||||||
|
.option(ChannelOption.AUTO_READ, false) |
||||||
|
.option(ChannelOption.TCP_NODELAY, true) |
||||||
|
} |
||||||
|
|
||||||
|
public fun createServerBootstrap(group: EventLoopGroup): ServerBootstrap { |
||||||
|
val channel = when (group) { |
||||||
|
is IOUringEventLoopGroup -> IOUringServerSocketChannel::class.java |
||||||
|
is EpollEventLoopGroup -> EpollServerSocketChannel::class.java |
||||||
|
is KQueueEventLoopGroup -> KQueueServerSocketChannel::class.java |
||||||
|
is NioEventLoopGroup -> NioServerSocketChannel::class.java |
||||||
|
else -> throw IllegalArgumentException("Unknown EventLoopGroup type") |
||||||
|
} |
||||||
|
|
||||||
|
return ServerBootstrap() |
||||||
|
.group(group) |
||||||
|
.channel(channel) |
||||||
|
.option(ChannelOption.ALLOCATOR, alloc) |
||||||
|
.option(ChannelOption.AUTO_READ, false) |
||||||
|
.childOption(ChannelOption.ALLOCATOR, alloc) |
||||||
|
.childOption(ChannelOption.AUTO_READ, false) |
||||||
|
.childOption(ChannelOption.TCP_NODELAY, true) |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,26 @@ |
|||||||
|
package dev.openrs2.net |
||||||
|
|
||||||
|
import io.netty.util.concurrent.Future |
||||||
|
import kotlin.coroutines.resume |
||||||
|
import kotlin.coroutines.resumeWithException |
||||||
|
import kotlin.coroutines.suspendCoroutine |
||||||
|
|
||||||
|
public suspend fun <V> Future<V>.suspend(): V { |
||||||
|
if (isDone) { |
||||||
|
if (isSuccess) { |
||||||
|
return now |
||||||
|
} else { |
||||||
|
throw cause() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return suspendCoroutine { continuation -> |
||||||
|
addListener { |
||||||
|
if (isSuccess) { |
||||||
|
continuation.resume(now) |
||||||
|
} else { |
||||||
|
continuation.resumeWithException(cause()) |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
@ -0,0 +1,10 @@ |
|||||||
|
package dev.openrs2.net |
||||||
|
|
||||||
|
import com.google.inject.AbstractModule |
||||||
|
import org.openrs2.buffer.BufferModule |
||||||
|
|
||||||
|
public object NetworkModule : AbstractModule() { |
||||||
|
override fun configure() { |
||||||
|
install(BufferModule) |
||||||
|
} |
||||||
|
} |
Loading…
Reference in new issue