diff --git a/backend/apps/library/e621.py b/backend/apps/library/e621.py index 98c4c50..adecd5b 100644 --- a/backend/apps/library/e621.py +++ b/backend/apps/library/e621.py @@ -13,7 +13,10 @@ import time import requests from django.conf import settings -REQUEST_INTERVAL = 0.5 # seconds between requests, per process +# Seconds between requests, per process. e621 allows 2/s hard and 1/s +# sustained; each gunicorn worker throttles on its own, so leave enough +# headroom that combined traffic does not trip the limit. +REQUEST_INTERVAL = 1.0 class E621Error(Exception): diff --git a/frontend/src/features/upload/UploadPage.tsx b/frontend/src/features/upload/UploadPage.tsx index 2e07dce..4591831 100644 --- a/frontend/src/features/upload/UploadPage.tsx +++ b/frontend/src/features/upload/UploadPage.tsx @@ -21,6 +21,7 @@ import { api, authHeaders, apiUrl, errorMessage, uploadFile } from "@/lib/api"; import { cn } from "@/lib/cn"; import { E621Error, + e621CooldownRemainingMs, e621ErrorMessage, effectiveCredentials, fetchPost, @@ -823,6 +824,15 @@ export default function UploadPage() { setIqdbError(null); try { while (iqdbQueueRef.current.length > 0) { + // Wait out a rate-limit cooldown before touching e621 again. + const cooldown = e621CooldownRemainingMs(); + if (cooldown > 0) { + setIqdbError( + `e621 rate limit — resuming in ${Math.ceil(cooldown / 1000)}s`, + ); + await new Promise((resolve) => setTimeout(resolve, cooldown + 250)); + setIqdbError(null); + } const tempId = iqdbQueueRef.current.shift() as string; const cached = queryClient.getQueryData(["uploads"]) ?? []; diff --git a/frontend/src/lib/e621.ts b/frontend/src/lib/e621.ts index fcecd60..ae7acb0 100644 --- a/frontend/src/lib/e621.ts +++ b/frontend/src/lib/e621.ts @@ -89,13 +89,50 @@ export function effectiveCredentials( const GIT_HASH = typeof __GIT_HASH__ === "string" ? __GIT_HASH__ : "dev"; const CLIENT_VERSION = `J621/${GIT_HASH} (JakeBreath)`; -// e621 allows 2 requests/second hard, 1/second sustained. Serialize every -// request through a queue with a 1s minimum gap. +// e621 allows 2 requests/second hard, 1/second sustained — and the IQDB +// endpoint is stricter, so stay comfortably under it. Serialize every request +// through a queue with a minimum gap. let lastRequestAt = 0; let queue: Promise = Promise.resolve(); /** A hung request would block the whole serialized queue forever. */ const REQUEST_TIMEOUT_MS = 20_000; +const REQUEST_GAP_MS = 1500; + +/** A 429 (or a CORS-blocked failure) pauses every e621 call for a while. */ +const RATE_LIMIT_COOLDOWN_MS = 60_000; +const COOLDOWN_KEY = "j621.e621.cooldown"; +let cooldownUntil = 0; + +function storedCooldown(): number { + try { + return Number(window.localStorage.getItem(COOLDOWN_KEY)) || 0; + } catch { + return 0; + } +} + +export function e621CooldownRemainingMs(): number { + return Math.max(0, Math.max(cooldownUntil, storedCooldown()) - Date.now()); +} + +function startCooldown(): void { + cooldownUntil = Date.now() + RATE_LIMIT_COOLDOWN_MS; + try { + window.localStorage.setItem(COOLDOWN_KEY, String(cooldownUntil)); + } catch { + // Storage disabled; the in-memory cooldown still applies. + } +} + +function isRateLimitError(error: unknown): boolean { + if (error instanceof E621Error) { + return error.status === 429 || error.status === 503; + } + // Browsers hide the status of a CORS-blocked response; a rate-limited 429 + // arrives here as a network error. + return error instanceof TypeError || error instanceof DOMException; +} function schedule(task: () => Promise): Promise { const run = queue.then(task, task); @@ -107,7 +144,11 @@ function schedule(task: () => Promise): Promise { } async function throttle(): Promise { - const wait = Math.max(0, lastRequestAt + 1000 - Date.now()); + const wait = Math.max( + 0, + lastRequestAt + REQUEST_GAP_MS - Date.now(), + e621CooldownRemainingMs(), + ); if (wait > 0) { await new Promise((resolve) => setTimeout(resolve, wait)); } @@ -177,6 +218,12 @@ export function e621Request( return undefined as T; } return (await response.json()) as T; + } catch (error) { + // Cancelled requests (navigation, unmounted query) are not rate limits. + if (!options.signal?.aborted && isRateLimitError(error)) { + startCooldown(); + } + throw error; } finally { clearTimeout(timer); options.signal?.removeEventListener("abort", forwardAbort); @@ -209,7 +256,7 @@ export function e621ErrorMessage(error: unknown): string { return "e621 did not answer in time."; } if (error instanceof TypeError) { - return "Could not reach e621. Check the base URL and your connection."; + return "Could not reach e621. If this keeps happening it is usually the rate limit — wait a minute and retry."; } return error instanceof Error ? error.message