From e2697c0a7827b6611bb27ee1911d557e52dab413 Mon Sep 17 00:00:00 2001
From: JakeBreath <tolozacdcmo@gmail.com>
Date: Tue, 22 Sep 2026 22:32:37 -0500
Subject: [PATCH] Stop throttling signed media and ease the browser's e621
 queue

Signed media URLs are fetched by <img>/<video> tags without an
Authorization header, so they were charged to the anonymous 120/min
bucket: past that, galleries and the fish-greeting download got 429 JSON
instead of image bytes. The raw/thumbnail/staged-file/similarity-file
actions are now exempt, and THROTTLE_ENABLED=false removes the general
anon+user limits for private/tailnet deployments (login/register/proxy
guards stay).

The SPA's e621 client also stops self-throttling so hard: 1s gap between
browsing calls (2.5s for the stricter IQDB endpoint) and a 15s cooldown
instead of 60s when e621 answers 429.
---
 backend/apps/core/tests/test_security.py | 30 ++++++++++++++++++++++++
 backend/apps/library/similarity.py       |  7 +++++-
 backend/apps/library/uploads.py          |  7 +++++-
 backend/apps/library/views.py            |  4 ++--
 backend/config/settings.py               | 27 +++++++++++++++++----
 deploy/.env.example                      |  4 ++++
 frontend/src/lib/e621.ts                 | 21 ++++++++++-------
 7 files changed, 82 insertions(+), 18 deletions(-)

diff --git a/backend/apps/core/tests/test_security.py b/backend/apps/core/tests/test_security.py
index 824bd6b..3151e08 100644
--- a/backend/apps/core/tests/test_security.py
+++ b/backend/apps/core/tests/test_security.py
@@ -429,6 +429,36 @@ class ThrottleTests(SecurityTestCase):
             {self.guest.get("/api/status/").status_code for _ in range(12)}, {200}
         )
 
+    def test_signed_media_urls_are_not_throttled(self):
+        """<img>/<video> tags fetch these without an Authorization header.
+
+        Regression: they were charged to the anonymous bucket, so galleries
+        and the fish-greeting download started returning 429 JSON instead of
+        the image bytes.
+        """
+        item = self.make_item("throttle-media", owner=self.users["sec-uploader"])
+        codes = {
+            self.guest.get(f"/api/files/J-{item.id}/raw/").status_code
+            for _ in range(150)
+        }
+        self.assertEqual(codes, {200})
+
+    def test_staged_upload_files_are_not_throttled(self):
+        temp = TempUpload.objects.create(
+            user=self.users["sec-uploader"],
+            file=SimpleUploadedFile("throttle-temp.bin", b"staged"),
+            original_filename="throttle-temp.bin",
+            md5=hashlib.md5(b"throttle-temp").hexdigest(),
+            size=6,
+        )
+        signature = signing.dumps(
+            {"temp": str(temp.id), "user": self.users["sec-uploader"].id},
+            salt=services.UPLOAD_FILE_SALT,
+        )
+        url = f"/api/uploads/{temp.id}/file/?sig={signature}"
+        codes = {self.guest.get(url).status_code for _ in range(150)}
+        self.assertEqual(codes, {200})
+
 
 class RemoteUrlTests(SecurityTestCase):
     def test_allowlist(self):
diff --git a/backend/apps/library/similarity.py b/backend/apps/library/similarity.py
index aee117e..90e02e2 100644
--- a/backend/apps/library/similarity.py
+++ b/backend/apps/library/similarity.py
@@ -136,7 +136,12 @@ class SimilarityCheckViewSet(
             self.get_serializer(check).data, status=status.HTTP_201_CREATED
         )
 
-    @action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
+    @action(
+        detail=True,
+        methods=["get", "head"],
+        permission_classes=[AllowAny],
+        throttle_classes=[],
+    )
     def file(self, request, pk=None):
         """Serve the temp file; accepts a signed URL like staged uploads."""
         check = None
diff --git a/backend/apps/library/uploads.py b/backend/apps/library/uploads.py
index 25fa899..d92b113 100644
--- a/backend/apps/library/uploads.py
+++ b/backend/apps/library/uploads.py
@@ -381,7 +381,12 @@ class TempUploadViewSet(
                 errors.append({"temp_id": value, "error": str(exc)})
         return Response({"discarded": discarded, "errors": errors})
 
-    @action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
+    @action(
+        detail=True,
+        methods=["get", "head"],
+        permission_classes=[AllowAny],
+        throttle_classes=[],
+    )
     def file(self, request, pk=None):
         """Serve the staged file; accepts a signed URL for media tags."""
         user = request.user if request.user.is_authenticated else None
diff --git a/backend/apps/library/views.py b/backend/apps/library/views.py
index 46819d3..5e3ee31 100644
--- a/backend/apps/library/views.py
+++ b/backend/apps/library/views.py
@@ -148,7 +148,7 @@ class MediaItemViewSet(
             return item
         return self.get_object()
 
-    @action(detail=True, methods=["get"])
+    @action(detail=True, methods=["get"], throttle_classes=[])
     def raw(self, request, pk=None):
         item = self._media_object(request, "raw")
         location = item.locations.first()
@@ -161,7 +161,7 @@ class MediaItemViewSet(
             request, location.path, download=request.query_params.get("download") == "1"
         )
 
-    @action(detail=True, methods=["get"])
+    @action(detail=True, methods=["get"], throttle_classes=[])
     def thumbnail(self, request, pk=None):
         item = self._media_object(request, "thumbnail")
         location = item.locations.first()
diff --git a/backend/config/settings.py b/backend/config/settings.py
index 8188541..b533fa1 100644
--- a/backend/config/settings.py
+++ b/backend/config/settings.py
@@ -255,6 +255,16 @@ CACHES = {
 
 # Django REST Framework
 
+# Private / tailnet-only deployments can drop the general anon+user limits
+# entirely (THROTTLE_ENABLED=false). The scoped guards below (login, register,
+# e621 proxy) and the media endpoints' own protections stay active either way.
+THROTTLE_ENABLED = os.getenv("THROTTLE_ENABLED", "true").strip().lower() not in {
+    "0",
+    "false",
+    "no",
+    "off",
+}
+
 REST_FRAMEWORK = {
     "DEFAULT_AUTHENTICATION_CLASSES": [
         "rest_framework.authentication.TokenAuthentication",
@@ -269,11 +279,18 @@ REST_FRAMEWORK = {
     ],
     "DEFAULT_PAGINATION_CLASS": "config.pagination.StandardPagination",
     "PAGE_SIZE": 48,
-    # Per-IP/per-user rate limits (counted in the shared Redis cache).
-    "DEFAULT_THROTTLE_CLASSES": [
-        "rest_framework.throttling.AnonRateThrottle",
-        "rest_framework.throttling.UserRateThrottle",
-    ],
+    # Per-IP/per-user rate limits (counted in the shared Redis cache). Signed
+    # media URLs are deliberately excluded at the view level: <img>/<video>
+    # tags fetch them without an Authorization header, so a library page would
+    # otherwise burn the anonymous bucket and start returning JSON 429s.
+    "DEFAULT_THROTTLE_CLASSES": (
+        [
+            "rest_framework.throttling.AnonRateThrottle",
+            "rest_framework.throttling.UserRateThrottle",
+        ]
+        if THROTTLE_ENABLED
+        else []
+    ),
     "DEFAULT_THROTTLE_RATES": {
         # Generous enough for the shell polling (status every 5s, stats every 2s).
         "anon": os.getenv("THROTTLE_ANON", "120/min"),
diff --git a/deploy/.env.example b/deploy/.env.example
index 17fc069..9862f7b 100644
--- a/deploy/.env.example
+++ b/deploy/.env.example
@@ -64,6 +64,10 @@ DB_ROOT_PASSWORD=j621root
 # THROTTLE_LOGIN=5/min
 # THROTTLE_REGISTER=20/hour
 # THROTTLE_E621_PROXY=60/hour
+# Tailnet-only / private deployments can drop the general limits entirely.
+# Signed media URLs (<img>/<video>) and the login/register/proxy guards are
+# exempt from this switch either way.
+# THROTTLE_ENABLED=false
 
 # e621 media hosts the backend may fetch from (downloads, proxies)
 # E621_MEDIA_HOSTS=static1.e621.net,static2.e621.net,static3.e621.net
diff --git a/frontend/src/lib/e621.ts b/frontend/src/lib/e621.ts
index 4027a02..70b990f 100644
--- a/frontend/src/lib/e621.ts
+++ b/frontend/src/lib/e621.ts
@@ -89,18 +89,19 @@ export function effectiveCredentials(
 const GIT_HASH = typeof __GIT_HASH__ === "string" ? __GIT_HASH__ : "dev";
 const CLIENT_VERSION = `J621/${GIT_HASH} (JakeBreath)`;
 
-// e621 allows 2 requests/second hard, 1/second sustained — and the IQDB
-// endpoint is stricter, so stay comfortably under it. Serialize every request
-// through a queue with a minimum gap.
+// e621 allows 2 requests/second hard, 1/second sustained. Serialize every
+// request through a queue with a minimum gap; IQDB is throttled much harder
+// by e621, so it gets a wider gap of its own.
 let lastRequestAt = 0;
 let queue: Promise<unknown> = Promise.resolve();
 
 /** A hung request would block the whole serialized queue forever. */
 const REQUEST_TIMEOUT_MS = 20_000;
-const REQUEST_GAP_MS = 1500;
+const REQUEST_GAP_MS = 1000;
+const IQDB_GAP_MS = 2500;
 
-/** A 429 (or a CORS-blocked failure) pauses every e621 call for a while. */
-const RATE_LIMIT_COOLDOWN_MS = 60_000;
+/** A 429 (or a CORS-blocked failure) pauses every e621 call briefly. */
+const RATE_LIMIT_COOLDOWN_MS = 15_000;
 const COOLDOWN_KEY = "j621.e621.cooldown";
 let cooldownUntil = 0;
 
@@ -143,10 +144,10 @@ function schedule<T>(task: () => Promise<T>): Promise<T> {
   return run;
 }
 
-async function throttle(): Promise<void> {
+async function throttle(minGap = REQUEST_GAP_MS): Promise<void> {
   const wait = Math.max(
     0,
-    lastRequestAt + REQUEST_GAP_MS - Date.now(),
+    lastRequestAt + minGap - Date.now(),
     e621CooldownRemainingMs(),
   );
   if (wait > 0) {
@@ -168,7 +169,9 @@ export function e621Request<T>(
   options: E621RequestOptions = {},
 ): Promise<T> {
   return schedule(async () => {
-    await throttle();
+    await throttle(
+      path.includes("iqdb_queries") ? IQDB_GAP_MS : REQUEST_GAP_MS,
+    );
 
     const base = credentials.base_url.replace(/\/+$/, "");
     const url = new URL(`${base}/${path.replace(/^\/+/, "")}`);