Make media URLs stable and cacheable, add real image thumbnails
Signed media URLs embedded the current second (TimestampSigner), so every API response re-minted every raw/thumbnail/staged URL and the browser re-downloaded each file on every poll or navigation. Responses also carried no cache headers at all. - sign with a plain Signer plus a bucket-quantized exp (7d TTL, 24h bucket), so a URL is byte-identical across responses and rotates once a day; legacy TimestampSigner URLs stay accepted for one release - add a v=<md5> version parameter to library media URLs so replacing a file under the same J-ID (the optimize flow) busts caches exactly when needed - serve_file now sends ETag/Last-Modified and a private Cache-Control and answers conditional requests with 304; library media gets max-age 6d + immutable, staged/similarity files 1h - build cached 480px JPEG thumbnails for images (Pillow, keyed by MD5 under MEDIA_ROOT/thumbs) instead of serving full-size originals through the thumbnail endpoint; the library grid uses thumbnail_url for images too
This commit is contained in:
@@ -36,6 +36,7 @@ from apps.library.models import (
|
|||||||
TempUpload,
|
TempUpload,
|
||||||
)
|
)
|
||||||
from apps.library.services import MEDIA_FILE_SALT
|
from apps.library.services import MEDIA_FILE_SALT
|
||||||
|
from apps.library.signing_urls import sign_payload
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
|
|
||||||
@@ -122,21 +123,15 @@ class SecurityTestCase(TestCase):
|
|||||||
return item
|
return item
|
||||||
|
|
||||||
def old_signature(self, item, action="raw", age=3 * 86400):
|
def old_signature(self, item, action="raw", age=3 * 86400):
|
||||||
"""A valid signature minted `age` seconds ago."""
|
"""A signed media URL whose expiry is `age` seconds in the past."""
|
||||||
real_time = signing.time
|
return signing.Signer(salt=MEDIA_FILE_SALT).sign_object(
|
||||||
|
{
|
||||||
class Backdated:
|
"item": item.id,
|
||||||
def time(self):
|
"user": self.users["sec-uploader"].id,
|
||||||
return real_time.time() - age
|
"action": action,
|
||||||
|
"exp": int(time.time()) - age,
|
||||||
try:
|
}
|
||||||
signing.time = Backdated()
|
)
|
||||||
return signing.dumps(
|
|
||||||
{"item": item.id, "user": self.users["sec-uploader"].id, "action": action},
|
|
||||||
salt=MEDIA_FILE_SALT,
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
signing.time = real_time
|
|
||||||
|
|
||||||
|
|
||||||
class GuestVisibilityTests(SecurityTestCase):
|
class GuestVisibilityTests(SecurityTestCase):
|
||||||
@@ -183,9 +178,9 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
def test_authenticated_users_and_signed_urls_see_protected_items(self):
|
def test_authenticated_users_and_signed_urls_see_protected_items(self):
|
||||||
uploader = self.client_for("sec-uploader")
|
uploader = self.client_for("sec-uploader")
|
||||||
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
|
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
|
||||||
signed = signing.dumps(
|
signed = sign_payload(
|
||||||
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
||||||
salt=MEDIA_FILE_SALT,
|
MEDIA_FILE_SALT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
|
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
|
||||||
@@ -193,9 +188,9 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_signature_integrity(self):
|
def test_signature_integrity(self):
|
||||||
signed = signing.dumps(
|
signed = sign_payload(
|
||||||
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
||||||
salt=MEDIA_FILE_SALT,
|
MEDIA_FILE_SALT,
|
||||||
)
|
)
|
||||||
raw = f"/api/files/J-{self.hidden.id}/raw/"
|
raw = f"/api/files/J-{self.hidden.id}/raw/"
|
||||||
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
|
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
|
||||||
@@ -203,10 +198,29 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
|
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
|
||||||
# Valid signature, wrong action.
|
# Valid signature, wrong action.
|
||||||
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
|
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
|
||||||
# Expired signature (minted three days ago).
|
# Expired signature (expiry three days ago).
|
||||||
expired = self.old_signature(self.hidden)
|
expired = self.old_signature(self.hidden)
|
||||||
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
|
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
|
||||||
|
|
||||||
|
def test_signed_media_urls_are_stable_and_versioned(self):
|
||||||
|
"""The same item must keep the same URL across responses.
|
||||||
|
|
||||||
|
A per-second signature made browsers re-download every image on every
|
||||||
|
poll; the MD5 version parameter busts caches only when the file itself
|
||||||
|
changes (the optimize flow rewrites files under the same J-ID).
|
||||||
|
"""
|
||||||
|
item = self.visible
|
||||||
|
first = services.signed_media_url(item, self.users["sec-uploader"])
|
||||||
|
time.sleep(1.1)
|
||||||
|
second = services.signed_media_url(item, self.users["sec-uploader"])
|
||||||
|
self.assertEqual(first, second)
|
||||||
|
self.assertIn(f"v={item.md5}", first)
|
||||||
|
MediaItem.objects.filter(pk=item.pk).update(md5="b" * 32)
|
||||||
|
item.refresh_from_db()
|
||||||
|
self.assertNotEqual(
|
||||||
|
services.signed_media_url(item, self.users["sec-uploader"]), first
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RoleBoundaryTests(SecurityTestCase):
|
class RoleBoundaryTests(SecurityTestCase):
|
||||||
def test_non_uploader_is_read_only(self):
|
def test_non_uploader_is_read_only(self):
|
||||||
@@ -451,9 +465,9 @@ class ThrottleTests(SecurityTestCase):
|
|||||||
md5=hashlib.md5(b"throttle-temp").hexdigest(),
|
md5=hashlib.md5(b"throttle-temp").hexdigest(),
|
||||||
size=6,
|
size=6,
|
||||||
)
|
)
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
|
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
|
||||||
salt=services.UPLOAD_FILE_SALT,
|
services.UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
|
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
|
||||||
codes = {self.guest.get(url).status_code for _ in range(150)}
|
codes = {self.guest.get(url).status_code for _ in range(150)}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from datetime import timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
@@ -20,6 +19,7 @@ from .services import (
|
|||||||
VIDEO_EXTENSIONS,
|
VIDEO_EXTENSIONS,
|
||||||
signed_media_url,
|
signed_media_url,
|
||||||
)
|
)
|
||||||
|
from .signing_urls import sign_payload
|
||||||
|
|
||||||
|
|
||||||
class MediaLocationSerializer(serializers.ModelSerializer):
|
class MediaLocationSerializer(serializers.ModelSerializer):
|
||||||
@@ -199,9 +199,9 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
user = self._request_user()
|
user = self._request_user()
|
||||||
if user is None:
|
if user is None:
|
||||||
return None
|
return None
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"temp": str(obj.id), "user": user.id},
|
{"temp": str(obj.id), "user": user.id},
|
||||||
salt=UPLOAD_FILE_SALT,
|
UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
|
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
|
||||||
request = self.context.get("request")
|
request = self.context.get("request")
|
||||||
@@ -339,9 +339,9 @@ class SimilarityCheckSerializer(serializers.ModelSerializer):
|
|||||||
user = self._request_user()
|
user = self._request_user()
|
||||||
if user is None or not obj.file:
|
if user is None or not obj.file:
|
||||||
return None
|
return None
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"check": str(obj.id), "user": user.id},
|
{"check": str(obj.id), "user": user.id},
|
||||||
salt=UPLOAD_FILE_SALT,
|
UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
|
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
|
||||||
request = self.context.get("request")
|
request = self.context.get("request")
|
||||||
|
|||||||
@@ -6,16 +6,20 @@ import os
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
import imagehash
|
import imagehash
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.http import FileResponse, Http404, HttpResponse
|
from django.http import FileResponse, Http404, HttpResponse
|
||||||
|
from django.utils.cache import get_conditional_response
|
||||||
|
from django.utils.http import http_date
|
||||||
from django.utils.text import get_valid_filename
|
from django.utils.text import get_valid_filename
|
||||||
from PIL import Image
|
from PIL import Image, ImageOps
|
||||||
|
|
||||||
from .models import MediaItem, MediaLocation
|
from .models import MediaItem, MediaLocation
|
||||||
|
from .signing_urls import sign_payload
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -37,6 +41,11 @@ UPLOAD_FILE_SALT = "j621.upload-file"
|
|||||||
MEDIA_FILE_SALT = "j621.media-file"
|
MEDIA_FILE_SALT = "j621.media-file"
|
||||||
CHUNK_SIZE = 1024 * 1024
|
CHUNK_SIZE = 1024 * 1024
|
||||||
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
||||||
|
# Versioned media URLs are immutable, so they may sit in the browser cache for
|
||||||
|
# as long as the signature is guaranteed to stay valid (7 days).
|
||||||
|
MEDIA_CACHE_SECONDS = 6 * 86400
|
||||||
|
# Staged uploads and similarity files can be deleted at any moment.
|
||||||
|
TEMP_CACHE_SECONDS = 3600
|
||||||
|
|
||||||
|
|
||||||
def compute_md5(path):
|
def compute_md5(path):
|
||||||
@@ -78,14 +87,24 @@ def signed_media_url(item, user, action="raw", request=None):
|
|||||||
|
|
||||||
With a ``request`` the URL is absolute, so the SPA also works when it is
|
With a ``request`` the URL is absolute, so the SPA also works when it is
|
||||||
served from a different origin; without one it stays relative.
|
served from a different origin; without one it stays relative.
|
||||||
|
|
||||||
|
The ``v`` parameter is the item's MD5: it busts the browser cache exactly
|
||||||
|
when the file is replaced (the optimize flow rewrites files under the same
|
||||||
|
J-ID), which is what lets the URL be cached for days instead of re-minted
|
||||||
|
on every response.
|
||||||
"""
|
"""
|
||||||
path = f"/api/files/J-{item.id}/{action}/"
|
path = f"/api/files/J-{item.id}/{action}/"
|
||||||
|
params = {}
|
||||||
if user is not None and getattr(user, "is_authenticated", False):
|
if user is not None and getattr(user, "is_authenticated", False):
|
||||||
signature = signing.dumps(
|
params = {
|
||||||
{"item": item.id, "user": user.id, "action": action},
|
"v": item.md5,
|
||||||
salt=MEDIA_FILE_SALT,
|
"sig": sign_payload(
|
||||||
)
|
{"item": item.id, "user": user.id, "action": action},
|
||||||
path = f"{path}?sig={signature}"
|
MEDIA_FILE_SALT,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if params:
|
||||||
|
path = f"{path}?{urlencode(params)}"
|
||||||
if request is None:
|
if request is None:
|
||||||
return path
|
return path
|
||||||
return request.build_absolute_uri(path)
|
return request.build_absolute_uri(path)
|
||||||
@@ -207,12 +226,36 @@ class RangeFileWrapper:
|
|||||||
self.file.close()
|
self.file.close()
|
||||||
|
|
||||||
|
|
||||||
def serve_file(request, path, download=False):
|
def _apply_cache_headers(response, cache_control, etag, mtime):
|
||||||
"""Serve a file with HTTP range support (needed for video seeking)."""
|
response["Cache-Control"] = cache_control
|
||||||
|
response["ETag"] = etag
|
||||||
|
response["Last-Modified"] = http_date(mtime)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def serve_file(request, path, download=False, *, max_age=TEMP_CACHE_SECONDS, immutable=False):
|
||||||
|
"""Serve a file with HTTP range support (needed for video seeking).
|
||||||
|
|
||||||
|
Responses carry validators (ETag/Last-Modified) and a private
|
||||||
|
``Cache-Control`` so browsers reuse media instead of re-downloading it on
|
||||||
|
every SPA poll. ``max_age``/``immutable`` are chosen by the caller: versioned
|
||||||
|
library media can be cached hard, staged files only briefly.
|
||||||
|
"""
|
||||||
path = Path(path)
|
path = Path(path)
|
||||||
if not path.is_file():
|
if not path.is_file():
|
||||||
raise Http404
|
raise Http404
|
||||||
size = path.stat().st_size
|
stat = path.stat()
|
||||||
|
size = stat.st_size
|
||||||
|
etag = f'W/"{size:x}-{stat.st_mtime_ns:x}"'
|
||||||
|
last_modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||||
|
conditional = get_conditional_response(
|
||||||
|
request, etag=etag, last_modified=last_modified
|
||||||
|
)
|
||||||
|
if conditional is not None:
|
||||||
|
return conditional
|
||||||
|
cache_control = f"private, max-age={int(max_age)}"
|
||||||
|
if immutable:
|
||||||
|
cache_control += ", immutable"
|
||||||
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
||||||
range_header = request.headers.get("Range", "").strip()
|
range_header = request.headers.get("Range", "").strip()
|
||||||
if range_header:
|
if range_header:
|
||||||
@@ -240,7 +283,9 @@ def serve_file(request, path, download=False):
|
|||||||
response["Content-Length"] = str(length)
|
response["Content-Length"] = str(length)
|
||||||
response["Content-Range"] = f"bytes {start}-{end}/{size}"
|
response["Content-Range"] = f"bytes {start}-{end}/{size}"
|
||||||
response["Accept-Ranges"] = "bytes"
|
response["Accept-Ranges"] = "bytes"
|
||||||
return response
|
return _apply_cache_headers(
|
||||||
|
response, cache_control, etag, stat.st_mtime
|
||||||
|
)
|
||||||
response = FileResponse(
|
response = FileResponse(
|
||||||
open(path, "rb"),
|
open(path, "rb"),
|
||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
@@ -248,7 +293,7 @@ def serve_file(request, path, download=False):
|
|||||||
filename=path.name,
|
filename=path.name,
|
||||||
)
|
)
|
||||||
response["Accept-Ranges"] = "bytes"
|
response["Accept-Ranges"] = "bytes"
|
||||||
return response
|
return _apply_cache_headers(response, cache_control, etag, stat.st_mtime)
|
||||||
|
|
||||||
|
|
||||||
class DownloadCancelled(Exception):
|
class DownloadCancelled(Exception):
|
||||||
@@ -465,3 +510,31 @@ def generate_video_thumbnail(md5, path):
|
|||||||
except (subprocess.SubprocessError, OSError):
|
except (subprocess.SubprocessError, OSError):
|
||||||
return None
|
return None
|
||||||
return target if target.exists() else None
|
return target if target.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
def generate_image_thumbnail(md5, path):
|
||||||
|
"""Downscale an image, cached under MEDIA_ROOT/thumbs like video thumbs.
|
||||||
|
|
||||||
|
The thumbnail action used to serve full-size originals for images; a
|
||||||
|
cached 480px JPEG keeps the library grid light without touching the
|
||||||
|
original file. Returns ``None`` when Pillow cannot decode the format, so
|
||||||
|
callers can fall back to the original.
|
||||||
|
"""
|
||||||
|
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
|
||||||
|
thumbs_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
target = thumbs_dir / f"{md5}.jpg"
|
||||||
|
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
|
||||||
|
return target
|
||||||
|
try:
|
||||||
|
with Image.open(path) as image:
|
||||||
|
# Animated formats: the first frame is the preview.
|
||||||
|
image.seek(0)
|
||||||
|
frame = ImageOps.exif_transpose(image) or image
|
||||||
|
frame = frame.convert("RGB")
|
||||||
|
frame.thumbnail((480, 480))
|
||||||
|
frame.save(target, "JPEG", quality=82, optimize=True)
|
||||||
|
except Exception: # noqa: BLE001 - previews must never break serving
|
||||||
|
logger.exception("Could not build an image thumbnail for %s", path)
|
||||||
|
target.unlink(missing_ok=True)
|
||||||
|
return None
|
||||||
|
return target if target.exists() else None
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""Stable, expiring signatures for media URLs.
|
||||||
|
|
||||||
|
The SPA loads media with ``<img>``/``<video>`` tags, which cannot send the
|
||||||
|
API's ``Authorization`` header, so those URLs carry a signature instead. The
|
||||||
|
signature has to be *stable*: a URL that changes on every response makes the
|
||||||
|
browser treat every refetch as a new resource and re-download the file.
|
||||||
|
|
||||||
|
URLs are signed with a plain ``Signer`` (no per-second timestamp) plus an
|
||||||
|
explicit ``exp`` claim quantized to a bucket, so every request inside a bucket
|
||||||
|
mints the exact same URL. The URL rotates once per bucket and is valid for at
|
||||||
|
least ``URL_TTL_SECONDS`` and at most ``URL_TTL_SECONDS + URL_BUCKET_SECONDS``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from django.core import signing
|
||||||
|
|
||||||
|
URL_TTL_SECONDS = 7 * 86400
|
||||||
|
URL_BUCKET_SECONDS = 24 * 3600
|
||||||
|
_BUCKETS = URL_TTL_SECONDS // URL_BUCKET_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def _expiry(now=None):
|
||||||
|
current = time.time() if now is None else now
|
||||||
|
bucket = int(current // URL_BUCKET_SECONDS)
|
||||||
|
return (bucket + _BUCKETS + 1) * URL_BUCKET_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def sign_payload(payload, salt, now=None):
|
||||||
|
"""Sign a payload with a stable, bucket-quantized expiry."""
|
||||||
|
return signing.Signer(salt=salt).sign_object(
|
||||||
|
{**payload, "exp": _expiry(now)}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_payload(signature, salt, legacy_max_age=86400):
|
||||||
|
"""Verify a signed payload; ``None`` when missing, tampered with or expired.
|
||||||
|
|
||||||
|
Signatures minted before the stable scheme (``TimestampSigner``) are still
|
||||||
|
accepted for one release so pages open across the deploy keep working.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = signing.Signer(salt=salt).unsign_object(signature)
|
||||||
|
except signing.BadSignature:
|
||||||
|
try:
|
||||||
|
return signing.TimestampSigner(salt=salt).unsign_object(
|
||||||
|
signature, max_age=legacy_max_age
|
||||||
|
)
|
||||||
|
except signing.BadSignature:
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
expired = int(data.get("exp", 0)) < time.time()
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return None if expired else data
|
||||||
@@ -11,7 +11,6 @@ from datetime import timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from rest_framework import mixins, status, viewsets
|
from rest_framework import mixins, status, viewsets
|
||||||
from rest_framework.decorators import action
|
from rest_framework.decorators import action
|
||||||
@@ -22,6 +21,7 @@ from rest_framework.response import Response
|
|||||||
from . import services
|
from . import services
|
||||||
from .models import MediaItem, SimilarityCheck
|
from .models import MediaItem, SimilarityCheck
|
||||||
from .serializers import SimilarityCheckSerializer
|
from .serializers import SimilarityCheckSerializer
|
||||||
|
from .signing_urls import load_payload
|
||||||
from .tools import item_brief
|
from .tools import item_brief
|
||||||
from .uploads import find_library_matches
|
from .uploads import find_library_matches
|
||||||
|
|
||||||
@@ -149,14 +149,7 @@ class SimilarityCheckViewSet(
|
|||||||
check = self.get_queryset().filter(pk=pk).first()
|
check = self.get_queryset().filter(pk=pk).first()
|
||||||
else:
|
else:
|
||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
payload = None
|
payload = load_payload(signature, services.UPLOAD_FILE_SALT) if signature else None
|
||||||
if signature:
|
|
||||||
try:
|
|
||||||
payload = signing.loads(
|
|
||||||
signature, salt=services.UPLOAD_FILE_SALT, max_age=86400
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
payload = None
|
|
||||||
if payload and payload.get("check") == str(pk):
|
if payload and payload.get("check") == str(pk):
|
||||||
check = SimilarityCheck.objects.filter(pk=pk).first()
|
check = SimilarityCheck.objects.filter(pk=pk).first()
|
||||||
if check is None or not check.file:
|
if check is None or not check.file:
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
"""Signed media URLs must be stable, versioned and cacheable.
|
||||||
|
|
||||||
|
Regression: signatures embedded the current second, so every API response
|
||||||
|
re-minted every URL and browsers re-downloaded each image on every poll; the
|
||||||
|
file responses also carried no cache headers at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
|
from django.test import Client, TestCase, override_settings
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from rest_framework.authtoken.models import Token
|
||||||
|
|
||||||
|
from apps.library import services
|
||||||
|
from apps.library.models import MediaItem, MediaLocation, TempUpload
|
||||||
|
from apps.library.signing_urls import sign_payload
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
TINY_PNG = base64.b64decode(
|
||||||
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def png_bytes(width=1200, height=800, color=(20, 120, 200)):
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
Image.new("RGB", (width, height), color).save(buffer, format="PNG")
|
||||||
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class MediaCacheTests(TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
super().setUpClass()
|
||||||
|
cls._tmp = tempfile.mkdtemp(prefix="j621-cache-")
|
||||||
|
cls._media = Path(cls._tmp) / "media"
|
||||||
|
cls._watched = cls._media / "library"
|
||||||
|
cls._watched.mkdir(parents=True, exist_ok=True)
|
||||||
|
cls._settings = override_settings(
|
||||||
|
MEDIA_ROOT=str(cls._media), WATCHED_FOLDER=str(cls._watched)
|
||||||
|
)
|
||||||
|
cls._settings.enable()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls._settings.disable()
|
||||||
|
shutil.rmtree(cls._tmp, ignore_errors=True)
|
||||||
|
super().tearDownClass()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(
|
||||||
|
username="cache-uploader", password="cache-pass-123456"
|
||||||
|
)
|
||||||
|
self.user.role = "uploader"
|
||||||
|
self.user.save(update_fields=["role"])
|
||||||
|
self.token = Token.objects.create(user=self.user).key
|
||||||
|
payload = png_bytes()
|
||||||
|
path = self._watched / "cache-image.png"
|
||||||
|
path.write_bytes(payload)
|
||||||
|
self.item = MediaItem.objects.create(
|
||||||
|
md5=hashlib.md5(payload).hexdigest(), size=len(payload)
|
||||||
|
)
|
||||||
|
MediaLocation.objects.create(
|
||||||
|
item=self.item, path=str(path), rel_path=path.name, mtime=time.time()
|
||||||
|
)
|
||||||
|
self.client = Client()
|
||||||
|
|
||||||
|
def signed(self, action):
|
||||||
|
return {
|
||||||
|
"sig": sign_payload(
|
||||||
|
{"item": self.item.id, "user": self.user.id, "action": action},
|
||||||
|
services.MEDIA_FILE_SALT,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_media_response_carries_cache_headers(self):
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/files/J-{self.item.id}/raw/", self.signed("raw")
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn("private", response["Cache-Control"])
|
||||||
|
self.assertIn(
|
||||||
|
f"max-age={services.MEDIA_CACHE_SECONDS}", response["Cache-Control"]
|
||||||
|
)
|
||||||
|
self.assertIn("immutable", response["Cache-Control"])
|
||||||
|
self.assertTrue(response["ETag"])
|
||||||
|
self.assertTrue(response["Last-Modified"])
|
||||||
|
|
||||||
|
def test_media_revalidation_returns_304(self):
|
||||||
|
url = f"/api/files/J-{self.item.id}/raw/"
|
||||||
|
first = self.client.get(url, self.signed("raw"))
|
||||||
|
second = self.client.get(
|
||||||
|
url, self.signed("raw"), HTTP_IF_NONE_MATCH=first["ETag"]
|
||||||
|
)
|
||||||
|
self.assertEqual(second.status_code, 304)
|
||||||
|
self.assertEqual(second.content, b"")
|
||||||
|
|
||||||
|
def test_image_thumbnail_is_generated_and_reused(self):
|
||||||
|
url = f"/api/files/J-{self.item.id}/thumbnail/"
|
||||||
|
response = self.client.get(url, self.signed("thumbnail"))
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response["Content-Type"], "image/jpeg")
|
||||||
|
thumb = self._media / "thumbs" / f"{self.item.md5}.jpg"
|
||||||
|
self.assertTrue(thumb.exists())
|
||||||
|
with Image.open(thumb) as image:
|
||||||
|
self.assertLessEqual(max(image.size), 480)
|
||||||
|
before = thumb.stat().st_mtime_ns
|
||||||
|
self.client.get(url, self.signed("thumbnail"))
|
||||||
|
self.assertEqual(thumb.stat().st_mtime_ns, before)
|
||||||
|
|
||||||
|
def test_staged_files_cache_briefly(self):
|
||||||
|
temp = TempUpload.objects.create(
|
||||||
|
user=self.user,
|
||||||
|
file=SimpleUploadedFile("staged.png", TINY_PNG, content_type="image/png"),
|
||||||
|
original_filename="staged.png",
|
||||||
|
md5=hashlib.md5(b"staged").hexdigest(),
|
||||||
|
size=len(TINY_PNG),
|
||||||
|
)
|
||||||
|
signature = sign_payload(
|
||||||
|
{"temp": str(temp.id), "user": self.user.id}, services.UPLOAD_FILE_SALT
|
||||||
|
)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/uploads/{temp.id}/file/", {"sig": signature}
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn(
|
||||||
|
f"max-age={services.TEMP_CACHE_SECONDS}", response["Cache-Control"]
|
||||||
|
)
|
||||||
|
self.assertNotIn("immutable", response["Cache-Control"])
|
||||||
@@ -16,7 +16,6 @@ from urllib.parse import urlparse
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
from django.core import signing
|
|
||||||
from django.core.exceptions import ValidationError
|
from django.core.exceptions import ValidationError
|
||||||
from django.http import Http404
|
from django.http import Http404
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
@@ -30,6 +29,7 @@ from . import services
|
|||||||
from .models import MediaItem, TempUpload
|
from .models import MediaItem, TempUpload
|
||||||
from .permissions import CanUpload
|
from .permissions import CanUpload
|
||||||
from .serializers import TempUploadListSerializer, TempUploadSerializer
|
from .serializers import TempUploadListSerializer, TempUploadSerializer
|
||||||
|
from .signing_urls import load_payload
|
||||||
from .tools import HASH_FIELDS, hashed_items, hashes_similarity
|
from .tools import HASH_FIELDS, hashed_items, hashes_similarity
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -393,14 +393,7 @@ class TempUploadViewSet(
|
|||||||
if user is None:
|
if user is None:
|
||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
if signature:
|
if signature:
|
||||||
try:
|
payload = load_payload(signature, services.UPLOAD_FILE_SALT)
|
||||||
payload = signing.loads(
|
|
||||||
signature,
|
|
||||||
salt=services.UPLOAD_FILE_SALT,
|
|
||||||
max_age=86400,
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
payload = None
|
|
||||||
if payload and str(payload.get("temp")) == str(pk):
|
if payload and str(payload.get("temp")) == str(pk):
|
||||||
user = (
|
user = (
|
||||||
get_user_model()
|
get_user_model()
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from pathlib import Path
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.db.models import Min, Q
|
from django.db.models import Min, Q
|
||||||
from django.http import Http404, StreamingHttpResponse
|
from django.http import Http404, StreamingHttpResponse
|
||||||
from django.shortcuts import get_object_or_404
|
from django.shortcuts import get_object_or_404
|
||||||
@@ -31,6 +30,7 @@ from .serializers import (
|
|||||||
MatchTaskSerializer,
|
MatchTaskSerializer,
|
||||||
MediaItemSerializer,
|
MediaItemSerializer,
|
||||||
)
|
)
|
||||||
|
from .signing_urls import load_payload
|
||||||
|
|
||||||
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
||||||
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
||||||
@@ -132,11 +132,8 @@ class MediaItemViewSet(
|
|||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
if not signature:
|
if not signature:
|
||||||
return None
|
return None
|
||||||
try:
|
payload = load_payload(signature, services.MEDIA_FILE_SALT)
|
||||||
payload = signing.loads(
|
if payload is None:
|
||||||
signature, salt=services.MEDIA_FILE_SALT, max_age=86400
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
return None
|
return None
|
||||||
if payload.get("action") != action_name:
|
if payload.get("action") != action_name:
|
||||||
return None
|
return None
|
||||||
@@ -158,7 +155,11 @@ class MediaItemViewSet(
|
|||||||
status=status.HTTP_404_NOT_FOUND,
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
)
|
)
|
||||||
return services.serve_file(
|
return services.serve_file(
|
||||||
request, location.path, download=request.query_params.get("download") == "1"
|
request,
|
||||||
|
location.path,
|
||||||
|
download=request.query_params.get("download") == "1",
|
||||||
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
@action(detail=True, methods=["get"], throttle_classes=[])
|
@action(detail=True, methods=["get"], throttle_classes=[])
|
||||||
@@ -173,13 +174,26 @@ class MediaItemViewSet(
|
|||||||
path = Path(location.path)
|
path = Path(location.path)
|
||||||
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
||||||
thumbnail = services.generate_video_thumbnail(item.md5, path)
|
thumbnail = services.generate_video_thumbnail(item.md5, path)
|
||||||
if thumbnail is None:
|
else:
|
||||||
return Response(
|
thumbnail = services.generate_image_thumbnail(item.md5, path)
|
||||||
{"detail": "Thumbnail unavailable."},
|
if thumbnail is not None:
|
||||||
status=status.HTTP_404_NOT_FOUND,
|
return services.serve_file(
|
||||||
)
|
request,
|
||||||
return services.serve_file(request, thumbnail)
|
thumbnail,
|
||||||
return services.serve_file(request, path)
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
|
)
|
||||||
|
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Thumbnail unavailable."},
|
||||||
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
|
)
|
||||||
|
return services.serve_file(
|
||||||
|
request,
|
||||||
|
path,
|
||||||
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
|
)
|
||||||
|
|
||||||
@action(detail=False, methods=["post"], permission_classes=[AllowAny])
|
@action(detail=False, methods=["post"], permission_classes=[AllowAny])
|
||||||
def lookup(self, request):
|
def lookup(self, request):
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ const ratingLabels: Record<string, string> = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export function MediaCard({ item }: { item: MediaItem }) {
|
export function MediaCard({ item }: { item: MediaItem }) {
|
||||||
const preview = apiUrl(
|
// The thumbnail endpoint now builds real 480px previews for images too, so
|
||||||
item.kind === "video" ? item.thumbnail_url : item.raw_url,
|
// the grid no longer pulls full-size originals.
|
||||||
);
|
const preview = apiUrl(item.thumbnail_url);
|
||||||
const rating = item.display_rating;
|
const rating = item.display_rating;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|||||||
Reference in New Issue
Block a user