Stop throttling signed media and ease the browser's e621 queue
Signed media URLs are fetched by <img>/<video> tags without an Authorization header, so they were charged to the anonymous 120/min bucket: past that, galleries and the fish-greeting download got 429 JSON instead of image bytes. The raw/thumbnail/staged-file/similarity-file actions are now exempt, and THROTTLE_ENABLED=false removes the general anon+user limits for private/tailnet deployments (login/register/proxy guards stay). The SPA's e621 client also stops self-throttling so hard: 1s gap between browsing calls (2.5s for the stricter IQDB endpoint) and a 15s cooldown instead of 60s when e621 answers 429.
This commit is contained in:
@@ -429,6 +429,36 @@ class ThrottleTests(SecurityTestCase):
|
||||
{self.guest.get("/api/status/").status_code for _ in range(12)}, {200}
|
||||
)
|
||||
|
||||
def test_signed_media_urls_are_not_throttled(self):
|
||||
"""<img>/<video> tags fetch these without an Authorization header.
|
||||
|
||||
Regression: they were charged to the anonymous bucket, so galleries
|
||||
and the fish-greeting download started returning 429 JSON instead of
|
||||
the image bytes.
|
||||
"""
|
||||
item = self.make_item("throttle-media", owner=self.users["sec-uploader"])
|
||||
codes = {
|
||||
self.guest.get(f"/api/files/J-{item.id}/raw/").status_code
|
||||
for _ in range(150)
|
||||
}
|
||||
self.assertEqual(codes, {200})
|
||||
|
||||
def test_staged_upload_files_are_not_throttled(self):
|
||||
temp = TempUpload.objects.create(
|
||||
user=self.users["sec-uploader"],
|
||||
file=SimpleUploadedFile("throttle-temp.bin", b"staged"),
|
||||
original_filename="throttle-temp.bin",
|
||||
md5=hashlib.md5(b"throttle-temp").hexdigest(),
|
||||
size=6,
|
||||
)
|
||||
signature = signing.dumps(
|
||||
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
|
||||
salt=services.UPLOAD_FILE_SALT,
|
||||
)
|
||||
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
|
||||
codes = {self.guest.get(url).status_code for _ in range(150)}
|
||||
self.assertEqual(codes, {200})
|
||||
|
||||
|
||||
class RemoteUrlTests(SecurityTestCase):
|
||||
def test_allowlist(self):
|
||||
|
||||
@@ -136,7 +136,12 @@ class SimilarityCheckViewSet(
|
||||
self.get_serializer(check).data, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get", "head"],
|
||||
permission_classes=[AllowAny],
|
||||
throttle_classes=[],
|
||||
)
|
||||
def file(self, request, pk=None):
|
||||
"""Serve the temp file; accepts a signed URL like staged uploads."""
|
||||
check = None
|
||||
|
||||
@@ -381,7 +381,12 @@ class TempUploadViewSet(
|
||||
errors.append({"temp_id": value, "error": str(exc)})
|
||||
return Response({"discarded": discarded, "errors": errors})
|
||||
|
||||
@action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
|
||||
@action(
|
||||
detail=True,
|
||||
methods=["get", "head"],
|
||||
permission_classes=[AllowAny],
|
||||
throttle_classes=[],
|
||||
)
|
||||
def file(self, request, pk=None):
|
||||
"""Serve the staged file; accepts a signed URL for media tags."""
|
||||
user = request.user if request.user.is_authenticated else None
|
||||
|
||||
@@ -148,7 +148,7 @@ class MediaItemViewSet(
|
||||
return item
|
||||
return self.get_object()
|
||||
|
||||
@action(detail=True, methods=["get"])
|
||||
@action(detail=True, methods=["get"], throttle_classes=[])
|
||||
def raw(self, request, pk=None):
|
||||
item = self._media_object(request, "raw")
|
||||
location = item.locations.first()
|
||||
@@ -161,7 +161,7 @@ class MediaItemViewSet(
|
||||
request, location.path, download=request.query_params.get("download") == "1"
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["get"])
|
||||
@action(detail=True, methods=["get"], throttle_classes=[])
|
||||
def thumbnail(self, request, pk=None):
|
||||
item = self._media_object(request, "thumbnail")
|
||||
location = item.locations.first()
|
||||
|
||||
@@ -255,6 +255,16 @@ CACHES = {
|
||||
|
||||
# Django REST Framework
|
||||
|
||||
# Private / tailnet-only deployments can drop the general anon+user limits
|
||||
# entirely (THROTTLE_ENABLED=false). The scoped guards below (login, register,
|
||||
# e621 proxy) and the media endpoints' own protections stay active either way.
|
||||
THROTTLE_ENABLED = os.getenv("THROTTLE_ENABLED", "true").strip().lower() not in {
|
||||
"0",
|
||||
"false",
|
||||
"no",
|
||||
"off",
|
||||
}
|
||||
|
||||
REST_FRAMEWORK = {
|
||||
"DEFAULT_AUTHENTICATION_CLASSES": [
|
||||
"rest_framework.authentication.TokenAuthentication",
|
||||
@@ -269,11 +279,18 @@ REST_FRAMEWORK = {
|
||||
],
|
||||
"DEFAULT_PAGINATION_CLASS": "config.pagination.StandardPagination",
|
||||
"PAGE_SIZE": 48,
|
||||
# Per-IP/per-user rate limits (counted in the shared Redis cache).
|
||||
"DEFAULT_THROTTLE_CLASSES": [
|
||||
"rest_framework.throttling.AnonRateThrottle",
|
||||
"rest_framework.throttling.UserRateThrottle",
|
||||
],
|
||||
# Per-IP/per-user rate limits (counted in the shared Redis cache). Signed
|
||||
# media URLs are deliberately excluded at the view level: <img>/<video>
|
||||
# tags fetch them without an Authorization header, so a library page would
|
||||
# otherwise burn the anonymous bucket and start returning JSON 429s.
|
||||
"DEFAULT_THROTTLE_CLASSES": (
|
||||
[
|
||||
"rest_framework.throttling.AnonRateThrottle",
|
||||
"rest_framework.throttling.UserRateThrottle",
|
||||
]
|
||||
if THROTTLE_ENABLED
|
||||
else []
|
||||
),
|
||||
"DEFAULT_THROTTLE_RATES": {
|
||||
# Generous enough for the shell polling (status every 5s, stats every 2s).
|
||||
"anon": os.getenv("THROTTLE_ANON", "120/min"),
|
||||
|
||||
Reference in New Issue
Block a user