diff --git a/backend/apps/core/tests/test_security.py b/backend/apps/core/tests/test_security.py
index 3151e08..e9f86f6 100644
--- a/backend/apps/core/tests/test_security.py
+++ b/backend/apps/core/tests/test_security.py
@@ -36,6 +36,7 @@ from apps.library.models import (
TempUpload,
)
from apps.library.services import MEDIA_FILE_SALT
+from apps.library.signing_urls import sign_payload
User = get_user_model()
@@ -122,21 +123,15 @@ class SecurityTestCase(TestCase):
return item
def old_signature(self, item, action="raw", age=3 * 86400):
- """A valid signature minted `age` seconds ago."""
- real_time = signing.time
-
- class Backdated:
- def time(self):
- return real_time.time() - age
-
- try:
- signing.time = Backdated()
- return signing.dumps(
- {"item": item.id, "user": self.users["sec-uploader"].id, "action": action},
- salt=MEDIA_FILE_SALT,
- )
- finally:
- signing.time = real_time
+ """A signed media URL whose expiry is `age` seconds in the past."""
+ return signing.Signer(salt=MEDIA_FILE_SALT).sign_object(
+ {
+ "item": item.id,
+ "user": self.users["sec-uploader"].id,
+ "action": action,
+ "exp": int(time.time()) - age,
+ }
+ )
class GuestVisibilityTests(SecurityTestCase):
@@ -183,9 +178,9 @@ class GuestVisibilityTests(SecurityTestCase):
def test_authenticated_users_and_signed_urls_see_protected_items(self):
uploader = self.client_for("sec-uploader")
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
- signed = signing.dumps(
+ signed = sign_payload(
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
- salt=MEDIA_FILE_SALT,
+ MEDIA_FILE_SALT,
)
self.assertEqual(
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
@@ -193,9 +188,9 @@ class GuestVisibilityTests(SecurityTestCase):
)
def test_signature_integrity(self):
- signed = signing.dumps(
+ signed = sign_payload(
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
- salt=MEDIA_FILE_SALT,
+ MEDIA_FILE_SALT,
)
raw = f"/api/files/J-{self.hidden.id}/raw/"
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
@@ -203,10 +198,29 @@ class GuestVisibilityTests(SecurityTestCase):
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
# Valid signature, wrong action.
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
- # Expired signature (minted three days ago).
+ # Expired signature (expiry three days ago).
expired = self.old_signature(self.hidden)
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
+ def test_signed_media_urls_are_stable_and_versioned(self):
+ """The same item must keep the same URL across responses.
+
+ A per-second signature made browsers re-download every image on every
+ poll; the MD5 version parameter busts caches only when the file itself
+ changes (the optimize flow rewrites files under the same J-ID).
+ """
+ item = self.visible
+ first = services.signed_media_url(item, self.users["sec-uploader"])
+ time.sleep(1.1)
+ second = services.signed_media_url(item, self.users["sec-uploader"])
+ self.assertEqual(first, second)
+ self.assertIn(f"v={item.md5}", first)
+ MediaItem.objects.filter(pk=item.pk).update(md5="b" * 32)
+ item.refresh_from_db()
+ self.assertNotEqual(
+ services.signed_media_url(item, self.users["sec-uploader"]), first
+ )
+
class RoleBoundaryTests(SecurityTestCase):
def test_non_uploader_is_read_only(self):
@@ -451,9 +465,9 @@ class ThrottleTests(SecurityTestCase):
md5=hashlib.md5(b"throttle-temp").hexdigest(),
size=6,
)
- signature = signing.dumps(
+ signature = sign_payload(
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
- salt=services.UPLOAD_FILE_SALT,
+ services.UPLOAD_FILE_SALT,
)
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
codes = {self.guest.get(url).status_code for _ in range(150)}
diff --git a/backend/apps/library/serializers.py b/backend/apps/library/serializers.py
index 55563db..2f0352f 100644
--- a/backend/apps/library/serializers.py
+++ b/backend/apps/library/serializers.py
@@ -3,7 +3,6 @@ from datetime import timedelta
from pathlib import Path
from django.conf import settings
-from django.core import signing
from rest_framework import serializers
from .models import (
@@ -20,6 +19,7 @@ from .services import (
VIDEO_EXTENSIONS,
signed_media_url,
)
+from .signing_urls import sign_payload
class MediaLocationSerializer(serializers.ModelSerializer):
@@ -199,9 +199,9 @@ class TempUploadSerializer(serializers.ModelSerializer):
user = self._request_user()
if user is None:
return None
- signature = signing.dumps(
+ signature = sign_payload(
{"temp": str(obj.id), "user": user.id},
- salt=UPLOAD_FILE_SALT,
+ UPLOAD_FILE_SALT,
)
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
request = self.context.get("request")
@@ -339,9 +339,9 @@ class SimilarityCheckSerializer(serializers.ModelSerializer):
user = self._request_user()
if user is None or not obj.file:
return None
- signature = signing.dumps(
+ signature = sign_payload(
{"check": str(obj.id), "user": user.id},
- salt=UPLOAD_FILE_SALT,
+ UPLOAD_FILE_SALT,
)
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
request = self.context.get("request")
diff --git a/backend/apps/library/services.py b/backend/apps/library/services.py
index 9ca3374..bdd0c64 100644
--- a/backend/apps/library/services.py
+++ b/backend/apps/library/services.py
@@ -6,16 +6,20 @@ import os
import re
import shutil
import subprocess
+from datetime import datetime, timezone
from pathlib import Path
+from urllib.parse import urlencode
import imagehash
from django.conf import settings
-from django.core import signing
from django.http import FileResponse, Http404, HttpResponse
+from django.utils.cache import get_conditional_response
+from django.utils.http import http_date
from django.utils.text import get_valid_filename
-from PIL import Image
+from PIL import Image, ImageOps
from .models import MediaItem, MediaLocation
+from .signing_urls import sign_payload
logger = logging.getLogger(__name__)
@@ -37,6 +41,11 @@ UPLOAD_FILE_SALT = "j621.upload-file"
MEDIA_FILE_SALT = "j621.media-file"
CHUNK_SIZE = 1024 * 1024
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
+# Versioned media URLs are immutable, so they may sit in the browser cache for
+# as long as the signature is guaranteed to stay valid (7 days).
+MEDIA_CACHE_SECONDS = 6 * 86400
+# Staged uploads and similarity files can be deleted at any moment.
+TEMP_CACHE_SECONDS = 3600
def compute_md5(path):
@@ -78,14 +87,24 @@ def signed_media_url(item, user, action="raw", request=None):
With a ``request`` the URL is absolute, so the SPA also works when it is
served from a different origin; without one it stays relative.
+
+ The ``v`` parameter is the item's MD5: it busts the browser cache exactly
+ when the file is replaced (the optimize flow rewrites files under the same
+ J-ID), which is what lets the URL be cached for days instead of re-minted
+ on every response.
"""
path = f"/api/files/J-{item.id}/{action}/"
+ params = {}
if user is not None and getattr(user, "is_authenticated", False):
- signature = signing.dumps(
- {"item": item.id, "user": user.id, "action": action},
- salt=MEDIA_FILE_SALT,
- )
- path = f"{path}?sig={signature}"
+ params = {
+ "v": item.md5,
+ "sig": sign_payload(
+ {"item": item.id, "user": user.id, "action": action},
+ MEDIA_FILE_SALT,
+ ),
+ }
+ if params:
+ path = f"{path}?{urlencode(params)}"
if request is None:
return path
return request.build_absolute_uri(path)
@@ -207,12 +226,36 @@ class RangeFileWrapper:
self.file.close()
-def serve_file(request, path, download=False):
- """Serve a file with HTTP range support (needed for video seeking)."""
+def _apply_cache_headers(response, cache_control, etag, mtime):
+ response["Cache-Control"] = cache_control
+ response["ETag"] = etag
+ response["Last-Modified"] = http_date(mtime)
+ return response
+
+
+def serve_file(request, path, download=False, *, max_age=TEMP_CACHE_SECONDS, immutable=False):
+ """Serve a file with HTTP range support (needed for video seeking).
+
+ Responses carry validators (ETag/Last-Modified) and a private
+ ``Cache-Control`` so browsers reuse media instead of re-downloading it on
+ every SPA poll. ``max_age``/``immutable`` are chosen by the caller: versioned
+ library media can be cached hard, staged files only briefly.
+ """
path = Path(path)
if not path.is_file():
raise Http404
- size = path.stat().st_size
+ stat = path.stat()
+ size = stat.st_size
+ etag = f'W/"{size:x}-{stat.st_mtime_ns:x}"'
+ last_modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
+ conditional = get_conditional_response(
+ request, etag=etag, last_modified=last_modified
+ )
+ if conditional is not None:
+ return conditional
+ cache_control = f"private, max-age={int(max_age)}"
+ if immutable:
+ cache_control += ", immutable"
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
range_header = request.headers.get("Range", "").strip()
if range_header:
@@ -240,7 +283,9 @@ def serve_file(request, path, download=False):
response["Content-Length"] = str(length)
response["Content-Range"] = f"bytes {start}-{end}/{size}"
response["Accept-Ranges"] = "bytes"
- return response
+ return _apply_cache_headers(
+ response, cache_control, etag, stat.st_mtime
+ )
response = FileResponse(
open(path, "rb"),
content_type=content_type,
@@ -248,7 +293,7 @@ def serve_file(request, path, download=False):
filename=path.name,
)
response["Accept-Ranges"] = "bytes"
- return response
+ return _apply_cache_headers(response, cache_control, etag, stat.st_mtime)
class DownloadCancelled(Exception):
@@ -465,3 +510,31 @@ def generate_video_thumbnail(md5, path):
except (subprocess.SubprocessError, OSError):
return None
return target if target.exists() else None
+
+
+def generate_image_thumbnail(md5, path):
+ """Downscale an image, cached under MEDIA_ROOT/thumbs like video thumbs.
+
+ The thumbnail action used to serve full-size originals for images; a
+ cached 480px JPEG keeps the library grid light without touching the
+ original file. Returns ``None`` when Pillow cannot decode the format, so
+ callers can fall back to the original.
+ """
+ thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
+ thumbs_dir.mkdir(parents=True, exist_ok=True)
+ target = thumbs_dir / f"{md5}.jpg"
+ if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
+ return target
+ try:
+ with Image.open(path) as image:
+ # Animated formats: the first frame is the preview.
+ image.seek(0)
+ frame = ImageOps.exif_transpose(image) or image
+ frame = frame.convert("RGB")
+ frame.thumbnail((480, 480))
+ frame.save(target, "JPEG", quality=82, optimize=True)
+ except Exception: # noqa: BLE001 - previews must never break serving
+ logger.exception("Could not build an image thumbnail for %s", path)
+ target.unlink(missing_ok=True)
+ return None
+ return target if target.exists() else None
diff --git a/backend/apps/library/signing_urls.py b/backend/apps/library/signing_urls.py
new file mode 100644
index 0000000..71bd0c1
--- /dev/null
+++ b/backend/apps/library/signing_urls.py
@@ -0,0 +1,57 @@
+"""Stable, expiring signatures for media URLs.
+
+The SPA loads media with ``
``/``