Make media URLs stable and cacheable, add real image thumbnails

Signed media URLs embedded the current second (TimestampSigner), so every
API response re-minted every raw/thumbnail/staged URL and the browser
re-downloaded each file on every poll or navigation. Responses also carried
no cache headers at all.

- sign with a plain Signer plus a bucket-quantized exp (7d TTL, 24h bucket),
  so a URL is byte-identical across responses and rotates once a day; legacy
  TimestampSigner URLs stay accepted for one release
- add a v=<md5> version parameter to library media URLs so replacing a file
  under the same J-ID (the optimize flow) busts caches exactly when needed
- serve_file now sends ETag/Last-Modified and a private Cache-Control and
  answers conditional requests with 304; library media gets max-age 6d +
  immutable, staged/similarity files 1h
- build cached 480px JPEG thumbnails for images (Pillow, keyed by MD5 under
  MEDIA_ROOT/thumbs) instead of serving full-size originals through the
  thumbnail endpoint; the library grid uses thumbnail_url for images too

backend/apps/library/serializers.py

@@ -3,7 +3,6 @@ from datetime import timedelta
 from pathlib import Path
 
 from django.conf import settings
-from django.core import signing
 from rest_framework import serializers
 
 from .models import (
@@ -20,6 +19,7 @@ from .services import (
     VIDEO_EXTENSIONS,
     signed_media_url,
 )
+from .signing_urls import sign_payload
 
 
 class MediaLocationSerializer(serializers.ModelSerializer):
@@ -199,9 +199,9 @@ class TempUploadSerializer(serializers.ModelSerializer):
         user = self._request_user()
         if user is None:
             return None
-        signature = signing.dumps(
+        signature = sign_payload(
             {"temp": str(obj.id), "user": user.id},
-            salt=UPLOAD_FILE_SALT,
+            UPLOAD_FILE_SALT,
         )
         url = f"/api/uploads/{obj.id}/file/?sig={signature}"
         request = self.context.get("request")
@@ -339,9 +339,9 @@ class SimilarityCheckSerializer(serializers.ModelSerializer):
         user = self._request_user()
         if user is None or not obj.file:
             return None
-        signature = signing.dumps(
+        signature = sign_payload(
             {"check": str(obj.id), "user": user.id},
-            salt=UPLOAD_FILE_SALT,
+            UPLOAD_FILE_SALT,
         )
         url = f"/api/similarity/{obj.id}/file/?sig={signature}"
         request = self.context.get("request")

backend/apps/library/services.py

@@ -6,16 +6,20 @@ import os
 import re
 import shutil
 import subprocess
+from datetime import datetime, timezone
 from pathlib import Path
+from urllib.parse import urlencode
 
 import imagehash
 from django.conf import settings
-from django.core import signing
 from django.http import FileResponse, Http404, HttpResponse
+from django.utils.cache import get_conditional_response
+from django.utils.http import http_date
 from django.utils.text import get_valid_filename
-from PIL import Image
+from PIL import Image, ImageOps
 
 from .models import MediaItem, MediaLocation
+from .signing_urls import sign_payload
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +41,11 @@ UPLOAD_FILE_SALT = "j621.upload-file"
 MEDIA_FILE_SALT = "j621.media-file"
 CHUNK_SIZE = 1024 * 1024
 RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
+# Versioned media URLs are immutable, so they may sit in the browser cache for
+# as long as the signature is guaranteed to stay valid (7 days).
+MEDIA_CACHE_SECONDS = 6 * 86400
+# Staged uploads and similarity files can be deleted at any moment.
+TEMP_CACHE_SECONDS = 3600
 
 
 def compute_md5(path):
@@ -78,14 +87,24 @@ def signed_media_url(item, user, action="raw", request=None):
 
     With a ``request`` the URL is absolute, so the SPA also works when it is
     served from a different origin; without one it stays relative.
+
+    The ``v`` parameter is the item's MD5: it busts the browser cache exactly
+    when the file is replaced (the optimize flow rewrites files under the same
+    J-ID), which is what lets the URL be cached for days instead of re-minted
+    on every response.
     """
     path = f"/api/files/J-{item.id}/{action}/"
+    params = {}
     if user is not None and getattr(user, "is_authenticated", False):
-        signature = signing.dumps(
-            {"item": item.id, "user": user.id, "action": action},
-            salt=MEDIA_FILE_SALT,
-        )
-        path = f"{path}?sig={signature}"
+        params = {
+            "v": item.md5,
+            "sig": sign_payload(
+                {"item": item.id, "user": user.id, "action": action},
+                MEDIA_FILE_SALT,
+            ),
+        }
+    if params:
+        path = f"{path}?{urlencode(params)}"
     if request is None:
         return path
     return request.build_absolute_uri(path)
@@ -207,12 +226,36 @@ class RangeFileWrapper:
         self.file.close()
 
 
-def serve_file(request, path, download=False):
-    """Serve a file with HTTP range support (needed for video seeking)."""
+def _apply_cache_headers(response, cache_control, etag, mtime):
+    response["Cache-Control"] = cache_control
+    response["ETag"] = etag
+    response["Last-Modified"] = http_date(mtime)
+    return response
+
+
+def serve_file(request, path, download=False, *, max_age=TEMP_CACHE_SECONDS, immutable=False):
+    """Serve a file with HTTP range support (needed for video seeking).
+
+    Responses carry validators (ETag/Last-Modified) and a private
+    ``Cache-Control`` so browsers reuse media instead of re-downloading it on
+    every SPA poll. ``max_age``/``immutable`` are chosen by the caller: versioned
+    library media can be cached hard, staged files only briefly.
+    """
     path = Path(path)
     if not path.is_file():
         raise Http404
-    size = path.stat().st_size
+    stat = path.stat()
+    size = stat.st_size
+    etag = f'W/"{size:x}-{stat.st_mtime_ns:x}"'
+    last_modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
+    conditional = get_conditional_response(
+        request, etag=etag, last_modified=last_modified
+    )
+    if conditional is not None:
+        return conditional
+    cache_control = f"private, max-age={int(max_age)}"
+    if immutable:
+        cache_control += ", immutable"
     content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
     range_header = request.headers.get("Range", "").strip()
     if range_header:
@@ -240,7 +283,9 @@ def serve_file(request, path, download=False):
             response["Content-Length"] = str(length)
             response["Content-Range"] = f"bytes {start}-{end}/{size}"
             response["Accept-Ranges"] = "bytes"
-            return response
+            return _apply_cache_headers(
+                response, cache_control, etag, stat.st_mtime
+            )
     response = FileResponse(
         open(path, "rb"),
         content_type=content_type,
@@ -248,7 +293,7 @@ def serve_file(request, path, download=False):
         filename=path.name,
     )
     response["Accept-Ranges"] = "bytes"
-    return response
+    return _apply_cache_headers(response, cache_control, etag, stat.st_mtime)
 
 
 class DownloadCancelled(Exception):
@@ -465,3 +510,31 @@ def generate_video_thumbnail(md5, path):
     except (subprocess.SubprocessError, OSError):
         return None
     return target if target.exists() else None
+
+
+def generate_image_thumbnail(md5, path):
+    """Downscale an image, cached under MEDIA_ROOT/thumbs like video thumbs.
+
+    The thumbnail action used to serve full-size originals for images; a
+    cached 480px JPEG keeps the library grid light without touching the
+    original file. Returns ``None`` when Pillow cannot decode the format, so
+    callers can fall back to the original.
+    """
+    thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
+    thumbs_dir.mkdir(parents=True, exist_ok=True)
+    target = thumbs_dir / f"{md5}.jpg"
+    if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
+        return target
+    try:
+        with Image.open(path) as image:
+            # Animated formats: the first frame is the preview.
+            image.seek(0)
+            frame = ImageOps.exif_transpose(image) or image
+            frame = frame.convert("RGB")
+            frame.thumbnail((480, 480))
+            frame.save(target, "JPEG", quality=82, optimize=True)
+    except Exception:  # noqa: BLE001 - previews must never break serving
+        logger.exception("Could not build an image thumbnail for %s", path)
+        target.unlink(missing_ok=True)
+        return None
+    return target if target.exists() else None

backend/apps/library/signing_urls.py

@@ -0,0 +1,57 @@
+"""Stable, expiring signatures for media URLs.
+
+The SPA loads media with ``<img>``/``<video>`` tags, which cannot send the
+API's ``Authorization`` header, so those URLs carry a signature instead. The
+signature has to be *stable*: a URL that changes on every response makes the
+browser treat every refetch as a new resource and re-download the file.
+
+URLs are signed with a plain ``Signer`` (no per-second timestamp) plus an
+explicit ``exp`` claim quantized to a bucket, so every request inside a bucket
+mints the exact same URL. The URL rotates once per bucket and is valid for at
+least ``URL_TTL_SECONDS`` and at most ``URL_TTL_SECONDS + URL_BUCKET_SECONDS``.
+"""
+
+import time
+
+from django.core import signing
+
+URL_TTL_SECONDS = 7 * 86400
+URL_BUCKET_SECONDS = 24 * 3600
+_BUCKETS = URL_TTL_SECONDS // URL_BUCKET_SECONDS
+
+
+def _expiry(now=None):
+    current = time.time() if now is None else now
+    bucket = int(current // URL_BUCKET_SECONDS)
+    return (bucket + _BUCKETS + 1) * URL_BUCKET_SECONDS
+
+
+def sign_payload(payload, salt, now=None):
+    """Sign a payload with a stable, bucket-quantized expiry."""
+    return signing.Signer(salt=salt).sign_object(
+        {**payload, "exp": _expiry(now)}
+    )
+
+
+def load_payload(signature, salt, legacy_max_age=86400):
+    """Verify a signed payload; ``None`` when missing, tampered with or expired.
+
+    Signatures minted before the stable scheme (``TimestampSigner``) are still
+    accepted for one release so pages open across the deploy keep working.
+    """
+    try:
+        data = signing.Signer(salt=salt).unsign_object(signature)
+    except signing.BadSignature:
+        try:
+            return signing.TimestampSigner(salt=salt).unsign_object(
+                signature, max_age=legacy_max_age
+            )
+        except signing.BadSignature:
+            return None
+    if not isinstance(data, dict):
+        return None
+    try:
+        expired = int(data.get("exp", 0)) < time.time()
+    except (TypeError, ValueError):
+        return None
+    return None if expired else data

backend/apps/library/similarity.py

@@ -11,7 +11,6 @@ from datetime import timedelta
 from pathlib import Path
 
 from django.conf import settings
-from django.core import signing
 from django.utils import timezone
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
@@ -22,6 +21,7 @@ from rest_framework.response import Response
 from . import services
 from .models import MediaItem, SimilarityCheck
 from .serializers import SimilarityCheckSerializer
+from .signing_urls import load_payload
 from .tools import item_brief
 from .uploads import find_library_matches
 
@@ -149,14 +149,7 @@ class SimilarityCheckViewSet(
             check = self.get_queryset().filter(pk=pk).first()
         else:
             signature = request.query_params.get("sig")
-            payload = None
-            if signature:
-                try:
-                    payload = signing.loads(
-                        signature, salt=services.UPLOAD_FILE_SALT, max_age=86400
-                    )
-                except signing.BadSignature:
-                    payload = None
+            payload = load_payload(signature, services.UPLOAD_FILE_SALT) if signature else None
             if payload and payload.get("check") == str(pk):
                 check = SimilarityCheck.objects.filter(pk=pk).first()
         if check is None or not check.file:

backend/apps/library/tests/test_media_cache.py

@@ -0,0 +1,139 @@
+"""Signed media URLs must be stable, versioned and cacheable.
+
+Regression: signatures embedded the current second, so every API response
+re-minted every URL and browsers re-downloaded each image on every poll; the
+file responses also carried no cache headers at all.
+"""
+
+import base64
+import hashlib
+import io
+import shutil
+import tempfile
+import time
+from pathlib import Path
+
+from django.contrib.auth import get_user_model
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.test import Client, TestCase, override_settings
+
+from PIL import Image
+
+from rest_framework.authtoken.models import Token
+
+from apps.library import services
+from apps.library.models import MediaItem, MediaLocation, TempUpload
+from apps.library.signing_urls import sign_payload
+
+User = get_user_model()
+
+TINY_PNG = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+)
+
+
+def png_bytes(width=1200, height=800, color=(20, 120, 200)):
+    buffer = io.BytesIO()
+    Image.new("RGB", (width, height), color).save(buffer, format="PNG")
+    return buffer.getvalue()
+
+
+class MediaCacheTests(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls._tmp = tempfile.mkdtemp(prefix="j621-cache-")
+        cls._media = Path(cls._tmp) / "media"
+        cls._watched = cls._media / "library"
+        cls._watched.mkdir(parents=True, exist_ok=True)
+        cls._settings = override_settings(
+            MEDIA_ROOT=str(cls._media), WATCHED_FOLDER=str(cls._watched)
+        )
+        cls._settings.enable()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls._settings.disable()
+        shutil.rmtree(cls._tmp, ignore_errors=True)
+        super().tearDownClass()
+
+    def setUp(self):
+        self.user = User.objects.create_user(
+            username="cache-uploader", password="cache-pass-123456"
+        )
+        self.user.role = "uploader"
+        self.user.save(update_fields=["role"])
+        self.token = Token.objects.create(user=self.user).key
+        payload = png_bytes()
+        path = self._watched / "cache-image.png"
+        path.write_bytes(payload)
+        self.item = MediaItem.objects.create(
+            md5=hashlib.md5(payload).hexdigest(), size=len(payload)
+        )
+        MediaLocation.objects.create(
+            item=self.item, path=str(path), rel_path=path.name, mtime=time.time()
+        )
+        self.client = Client()
+
+    def signed(self, action):
+        return {
+            "sig": sign_payload(
+                {"item": self.item.id, "user": self.user.id, "action": action},
+                services.MEDIA_FILE_SALT,
+            )
+        }
+
+    def test_media_response_carries_cache_headers(self):
+        response = self.client.get(
+            f"/api/files/J-{self.item.id}/raw/", self.signed("raw")
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("private", response["Cache-Control"])
+        self.assertIn(
+            f"max-age={services.MEDIA_CACHE_SECONDS}", response["Cache-Control"]
+        )
+        self.assertIn("immutable", response["Cache-Control"])
+        self.assertTrue(response["ETag"])
+        self.assertTrue(response["Last-Modified"])
+
+    def test_media_revalidation_returns_304(self):
+        url = f"/api/files/J-{self.item.id}/raw/"
+        first = self.client.get(url, self.signed("raw"))
+        second = self.client.get(
+            url, self.signed("raw"), HTTP_IF_NONE_MATCH=first["ETag"]
+        )
+        self.assertEqual(second.status_code, 304)
+        self.assertEqual(second.content, b"")
+
+    def test_image_thumbnail_is_generated_and_reused(self):
+        url = f"/api/files/J-{self.item.id}/thumbnail/"
+        response = self.client.get(url, self.signed("thumbnail"))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response["Content-Type"], "image/jpeg")
+        thumb = self._media / "thumbs" / f"{self.item.md5}.jpg"
+        self.assertTrue(thumb.exists())
+        with Image.open(thumb) as image:
+            self.assertLessEqual(max(image.size), 480)
+        before = thumb.stat().st_mtime_ns
+        self.client.get(url, self.signed("thumbnail"))
+        self.assertEqual(thumb.stat().st_mtime_ns, before)
+
+    def test_staged_files_cache_briefly(self):
+        temp = TempUpload.objects.create(
+            user=self.user,
+            file=SimpleUploadedFile("staged.png", TINY_PNG, content_type="image/png"),
+            original_filename="staged.png",
+            md5=hashlib.md5(b"staged").hexdigest(),
+            size=len(TINY_PNG),
+        )
+        signature = sign_payload(
+            {"temp": str(temp.id), "user": self.user.id}, services.UPLOAD_FILE_SALT
+        )
+        response = self.client.get(
+            f"/api/uploads/{temp.id}/file/", {"sig": signature}
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(
+            f"max-age={services.TEMP_CACHE_SECONDS}", response["Cache-Control"]
+        )
+        self.assertNotIn("immutable", response["Cache-Control"])

backend/apps/library/uploads.py

@@ -16,7 +16,6 @@ from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.core import signing
 from django.core.exceptions import ValidationError
 from django.http import Http404
 from django.utils import timezone
@@ -30,6 +29,7 @@ from . import services
 from .models import MediaItem, TempUpload
 from .permissions import CanUpload
 from .serializers import TempUploadListSerializer, TempUploadSerializer
+from .signing_urls import load_payload
 from .tools import HASH_FIELDS, hashed_items, hashes_similarity
 
 logger = logging.getLogger(__name__)
@@ -393,14 +393,7 @@ class TempUploadViewSet(
         if user is None:
             signature = request.query_params.get("sig")
             if signature:
-                try:
-                    payload = signing.loads(
-                        signature,
-                        salt=services.UPLOAD_FILE_SALT,
-                        max_age=86400,
-                    )
-                except signing.BadSignature:
-                    payload = None
+                payload = load_payload(signature, services.UPLOAD_FILE_SALT)
                 if payload and str(payload.get("temp")) == str(pk):
                     user = (
                         get_user_model()

backend/apps/library/views.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from django.conf import settings
-from django.core import signing
 from django.db.models import Min, Q
 from django.http import Http404, StreamingHttpResponse
 from django.shortcuts import get_object_or_404
@@ -31,6 +30,7 @@ from .serializers import (
     MatchTaskSerializer,
     MediaItemSerializer,
 )
+from .signing_urls import load_payload
 
 LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
 MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
@@ -132,11 +132,8 @@ class MediaItemViewSet(
         signature = request.query_params.get("sig")
         if not signature:
             return None
-        try:
-            payload = signing.loads(
-                signature, salt=services.MEDIA_FILE_SALT, max_age=86400
-            )
-        except signing.BadSignature:
+        payload = load_payload(signature, services.MEDIA_FILE_SALT)
+        if payload is None:
             return None
         if payload.get("action") != action_name:
             return None
@@ -158,7 +155,11 @@ class MediaItemViewSet(
                 status=status.HTTP_404_NOT_FOUND,
             )
         return services.serve_file(
-            request, location.path, download=request.query_params.get("download") == "1"
+            request,
+            location.path,
+            download=request.query_params.get("download") == "1",
+            max_age=services.MEDIA_CACHE_SECONDS,
+            immutable=True,
         )
 
     @action(detail=True, methods=["get"], throttle_classes=[])
@@ -173,13 +174,26 @@ class MediaItemViewSet(
         path = Path(location.path)
         if path.suffix.lower() in services.VIDEO_EXTENSIONS:
             thumbnail = services.generate_video_thumbnail(item.md5, path)
-            if thumbnail is None:
-                return Response(
-                    {"detail": "Thumbnail unavailable."},
-                    status=status.HTTP_404_NOT_FOUND,
-                )
-            return services.serve_file(request, thumbnail)
-        return services.serve_file(request, path)
+        else:
+            thumbnail = services.generate_image_thumbnail(item.md5, path)
+        if thumbnail is not None:
+            return services.serve_file(
+                request,
+                thumbnail,
+                max_age=services.MEDIA_CACHE_SECONDS,
+                immutable=True,
+            )
+        if path.suffix.lower() in services.VIDEO_EXTENSIONS:
+            return Response(
+                {"detail": "Thumbnail unavailable."},
+                status=status.HTTP_404_NOT_FOUND,
+            )
+        return services.serve_file(
+            request,
+            path,
+            max_age=services.MEDIA_CACHE_SECONDS,
+            immutable=True,
+        )
 
     @action(detail=False, methods=["post"], permission_classes=[AllowAny])
     def lookup(self, request):