From 26275fa0d75713a27e740f3b6804fe306e63e385 Mon Sep 17 00:00:00 2001 From: q13x <84165981+WorldEditAxe@users.noreply.github.com> Date: Thu, 21 Mar 2024 01:06:42 -0700 Subject: [PATCH] fix some bugs --- src/config.ts | 4 +- src/proxy/databases/DiskDB.ts | 3 +- .../ExponentialBackoffRequestController.ts | 63 +++++++++++++++++++ src/proxy/skins/EaglerSkins.ts | 20 +++++- src/proxy/skins/SkinServer.ts | 12 ++-- 5 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 src/proxy/ratelimit/ExponentialBackoffRequestController.ts diff --git a/src/config.ts b/src/config.ts index 08734ba..5aa3e97 100644 --- a/src/config.ts +++ b/src/config.ts @@ -15,8 +15,8 @@ export const config: Config = { cache: { useCache: true, folderName: "skinCache", - skinCacheLifetime: 60 * 1000, - skinCachePruneInterval: 5000, + skinCacheLifetime: 60 * 60 * 1000, + skinCachePruneInterval: 10 * 60 * 1000, }, }, motd: true diff --git a/src/proxy/databases/DiskDB.ts b/src/proxy/databases/DiskDB.ts index 04d24df..89430dc 100644 --- a/src/proxy/databases/DiskDB.ts +++ b/src/proxy/databases/DiskDB.ts @@ -20,7 +20,8 @@ export default class DiskDB { public async filter(f: (v: T) => boolean) { for (const file of await fs.readdir(this.folder)) { - if (!f(this.decoder(await fs.readFile(file)))) await fs.rm(file); + const fp = path.join(this.folder, file); + if (!f(this.decoder(await fs.readFile(fp)))) await fs.rm(fp); } } diff --git a/src/proxy/ratelimit/ExponentialBackoffRequestController.ts b/src/proxy/ratelimit/ExponentialBackoffRequestController.ts new file mode 100644 index 0000000..6ee4d12 --- /dev/null +++ b/src/proxy/ratelimit/ExponentialBackoffRequestController.ts @@ -0,0 +1,63 @@ +const wait = (ms: number) => new Promise((res) => setTimeout(res, ms)); + +export default class ExponentialBackoffRequestController { + public queue: Task[]; + public flushQueueAfterTries: number; + public baseDelay: number; + ended: boolean; + aborted: boolean; + + constructor(baseDelay: number = 3000, triesBeforeFlush: number = 10) { + this.flushQueueAfterTries = triesBeforeFlush; + this.baseDelay = baseDelay; + this.queue = []; + this.ended = false; + this.aborted = false; + setTimeout(() => this.tick(), 0); + } + + private async tick() { + while (true) { + if (this.ended) break; + for (const task of this.queue) { + if (this.ended || this.aborted) break; + let times = 0, + breakOut = false; + while (true) { + try { + await task(); + break; + } catch (err) { + times++; + await wait(this.baseDelay * 2 ** times); + if (times > this.flushQueueAfterTries) { + this.queue.forEach((task) => task(new Error("Controller overload!"))); + breakOut = true; + break; + } + } + } + if (breakOut) break; + } + if (this.aborted) this.aborted = false; + this.queue = []; + await wait(1); + } + } + + public end() { + this.ended = true; + } + + public flush() { + this.aborted = false; + this.queue.forEach((task) => task(new Error("Aborted"))); + this.queue = []; + } + + public queueTask(task: Task): void { + this.queue.push(task); + } +} + +type Task = (err?: object) => void | Promise; diff --git a/src/proxy/skins/EaglerSkins.ts b/src/proxy/skins/EaglerSkins.ts index c3f8e7c..bb6c8f2 100644 --- a/src/proxy/skins/EaglerSkins.ts +++ b/src/proxy/skins/EaglerSkins.ts @@ -11,6 +11,7 @@ import { Logger } from "../../logger.js"; import fetch from "node-fetch"; import Jimp from "jimp"; import { ImageEditor } from "./ImageEditor.js"; +import ExponentialBackoffRequestController from "../ratelimit/ExponentialBackoffRequestController.js"; // TODO: convert all functions to use MineProtocol's UUID manipulation functions @@ -81,7 +82,10 @@ export namespace EaglerSkins { return new Promise(async (res, rej) => { const skin = await fetch(skinUrl); if (skin.status != 200) { - rej(`Tried to fetch ${skinUrl}, got HTTP ${skin.status} instead!`); + rej({ + url: skinUrl, + status: skin.status, + }); return; } else { res(Buffer.from(await skin.arrayBuffer())); @@ -89,6 +93,20 @@ export namespace EaglerSkins { }); } + export function safeDownloadSkin(skinUrl: string, backoff: ExponentialBackoffRequestController): Promise { + return new Promise((res, rej) => { + backoff.queueTask(async (err) => { + if (err) return rej(err); + try { + res(await downloadSkin(skinUrl)); + } catch (err) { + if (err.status == 429) throw new Error("Ratelimited!"); + else rej("Unexpected HTTP status code: " + err.status); + } + }); + }); + } + export function readClientDownloadSkinRequestPacket(message: Buffer): ClientDownloadSkinRequest { const ret: ClientDownloadSkinRequest = { id: null, diff --git a/src/proxy/skins/SkinServer.ts b/src/proxy/skins/SkinServer.ts index 5237f76..4944c19 100644 --- a/src/proxy/skins/SkinServer.ts +++ b/src/proxy/skins/SkinServer.ts @@ -11,11 +11,13 @@ import { SCChannelMessagePacket } from "../packets/channel/SCChannelMessage.js"; import { EaglerSkins } from "./EaglerSkins.js"; import { ImageEditor } from "./ImageEditor.js"; import { MineProtocol } from "../Protocol.js"; +import ExponentialBackoffRequestController from "../ratelimit/ExponentialBackoffRequestController.js"; export class SkinServer { public allowedSkinDomains: string[]; public cache: DiskDB; public proxy: Proxy; + public backoffController: ExponentialBackoffRequestController; public usingNative: boolean; public usingCache: boolean; private _logger: Logger; @@ -29,13 +31,14 @@ export class SkinServer { cacheFolder, (v) => exportCachedSkin(v), (b) => readCachedSkin(b), - (k) => k + (k) => k.replaceAll("-", "") ); } this.proxy = proxy ?? PROXY; this.usingCache = useCache; this.usingNative = native; this.lifetime = cacheLifetime; + this.backoffController = new ExponentialBackoffRequestController(); this._logger = new Logger("SkinServer"); this._logger.info("Started EaglercraftX skin server."); if (useCache) this.deleteTask = setInterval(async () => await this.cache.filter((ent) => Date.now() < ent.expires), sweepInterval); @@ -88,17 +91,16 @@ export class SkinServer { skin = null; if (this.usingCache) { (cacheHit = await this.cache.get(parsedPacket_1.uuid)), (skin = cacheHit != null ? cacheHit.data : null); + if (!skin) { - this._logger.info("cache miss: getting skin"); - const fetched = await EaglerSkins.downloadSkin(parsedPacket_1.url); + const fetched = await EaglerSkins.safeDownloadSkin(parsedPacket_1.url, this.backoffController); skin = fetched; await this.cache.set(parsedPacket_1.uuid, { uuid: parsedPacket_1.uuid, expires: Date.now() + this.lifetime, data: fetched, }); - this._logger.info("downloaded skin, saved to cache"); - } else this._logger.info("hit success: got skin"); + } } else { skin = await EaglerSkins.downloadSkin(parsedPacket_1.url); }