Make thumbnail generation atomic and warm it on import

- write thumbnails to a .part file and os.replace() them, so concurrent
  requests never read a half-written JPEG
- a stale thumbnail plus a vanished source no longer raises through the
  request (getmtime on a missing file returned 500); it falls back cleanly
- ensure_thumbnail(item) warms the preview when a file is indexed, keeping
  image decoding out of the request path
This commit is contained in:
2026-09-23 21:35:09 -05:00
parent a62195ffce
commit 7c2569522f
3 changed files with 87 additions and 11 deletions
+60 -11
View File
@@ -6,6 +6,7 @@ import os
import re import re
import shutil import shutil
import subprocess import subprocess
import uuid
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from urllib.parse import urlencode from urllib.parse import urlencode
@@ -481,15 +482,38 @@ def sanitize_iqdb_results(results):
return cleaned return cleaned
def _thumbnail_is_fresh(target, path):
"""True when the cached thumbnail exists and is at least as new as source."""
try:
stat = target.stat()
if stat.st_size <= 0:
return False
return stat.st_mtime >= os.path.getmtime(path)
except OSError:
return False
def _thumbs_dir():
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
thumbs_dir.mkdir(parents=True, exist_ok=True)
return thumbs_dir
def generate_video_thumbnail(md5, path): def generate_video_thumbnail(md5, path):
"""Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs.""" """Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
if not shutil.which("ffmpeg"): if not shutil.which("ffmpeg"):
return None return None
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs" try:
thumbs_dir.mkdir(parents=True, exist_ok=True) thumbs_dir = _thumbs_dir()
except OSError:
logger.exception("Could not create the thumbnail folder")
return None
target = thumbs_dir / f"{md5}.jpg" target = thumbs_dir / f"{md5}.jpg"
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path): if _thumbnail_is_fresh(target, path):
return target return target
# Write beside the target and move it into place, so a concurrent request
# can never read a half-written JPEG.
temp = thumbs_dir / f".{md5}.{uuid.uuid4().hex}.part.jpg"
command = [ command = [
"ffmpeg", "ffmpeg",
"-y", "-y",
@@ -503,11 +527,14 @@ def generate_video_thumbnail(md5, path):
"scale=480:-2", "scale=480:-2",
"-loglevel", "-loglevel",
"error", "error",
str(target), str(temp),
] ]
try: try:
subprocess.run(command, check=True, capture_output=True, timeout=60) subprocess.run(command, check=True, capture_output=True, timeout=60)
os.replace(temp, target)
except (subprocess.SubprocessError, OSError): except (subprocess.SubprocessError, OSError):
logger.exception("Could not build a video thumbnail for %s", path)
temp.unlink(missing_ok=True)
return None return None
return target if target.exists() else None return target if target.exists() else None
@@ -517,14 +544,18 @@ def generate_image_thumbnail(md5, path):
The thumbnail action used to serve full-size originals for images; a The thumbnail action used to serve full-size originals for images; a
cached 480px JPEG keeps the library grid light without touching the cached 480px JPEG keeps the library grid light without touching the
original file. Returns ``None`` when Pillow cannot decode the format, so original file. Returns ``None`` when the source is missing or Pillow
callers can fall back to the original. cannot decode it, so callers can fall back to the original.
""" """
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs" try:
thumbs_dir.mkdir(parents=True, exist_ok=True) thumbs_dir = _thumbs_dir()
except OSError:
logger.exception("Could not create the thumbnail folder")
return None
target = thumbs_dir / f"{md5}.jpg" target = thumbs_dir / f"{md5}.jpg"
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path): if _thumbnail_is_fresh(target, path):
return target return target
temp = thumbs_dir / f".{md5}.{uuid.uuid4().hex}.part.jpg"
try: try:
with Image.open(path) as image: with Image.open(path) as image:
# Animated formats: the first frame is the preview. # Animated formats: the first frame is the preview.
@@ -532,9 +563,27 @@ def generate_image_thumbnail(md5, path):
frame = ImageOps.exif_transpose(image) or image frame = ImageOps.exif_transpose(image) or image
frame = frame.convert("RGB") frame = frame.convert("RGB")
frame.thumbnail((480, 480)) frame.thumbnail((480, 480))
frame.save(target, "JPEG", quality=82, optimize=True) frame.save(temp, "JPEG", quality=82, optimize=True)
os.replace(temp, target)
except Exception: # noqa: BLE001 - previews must never break serving except Exception: # noqa: BLE001 - previews must never break serving
logger.exception("Could not build an image thumbnail for %s", path) logger.exception("Could not build an image thumbnail for %s", path)
target.unlink(missing_ok=True) temp.unlink(missing_ok=True)
return None return None
return target if target.exists() else None return target if target.exists() else None
def ensure_thumbnail(item):
"""Generate an item's cached thumbnail if it is missing or stale.
Warming thumbnails when a file is indexed keeps image decoding out of the
request path, where the upload pipeline's hashing used to starve it.
"""
location = item.locations.first()
if location is None:
return None
path = Path(location.path)
if not path.is_file():
return None
if path.suffix.lower() in VIDEO_EXTENSIONS:
return generate_video_thumbnail(item.md5, path)
return generate_image_thumbnail(item.md5, path)
@@ -120,6 +120,27 @@ class MediaCacheTests(TestCase):
self.client.get(url, self.signed("thumbnail")) self.client.get(url, self.signed("thumbnail"))
self.assertEqual(thumb.stat().st_mtime_ns, before) self.assertEqual(thumb.stat().st_mtime_ns, before)
def test_ensure_thumbnail_reuses_the_cache(self):
first = services.ensure_thumbnail(self.item)
self.assertIsNotNone(first)
self.assertTrue(first.exists())
mtime = first.stat().st_mtime_ns
second = services.ensure_thumbnail(self.item)
self.assertEqual(second, first)
self.assertEqual(second.stat().st_mtime_ns, mtime)
def test_thumbnail_of_a_missing_source_does_not_error(self):
"""Regression: getmtime() on a vanished source used to raise a 500."""
thumbs = self._media / "thumbs"
thumbs.mkdir(parents=True, exist_ok=True)
(thumbs / f"{self.item.md5}.jpg").write_bytes(b"stale")
Path(self.item.locations.first().path).unlink()
self.assertIsNone(services.ensure_thumbnail(self.item))
response = self.client.get(
f"/api/files/J-{self.item.id}/thumbnail/", self.signed("thumbnail")
)
self.assertEqual(response.status_code, 404)
def test_staged_files_cache_briefly(self): def test_staged_files_cache_briefly(self):
temp = TempUpload.objects.create( temp = TempUpload.objects.create(
user=self.user, user=self.user,
+6
View File
@@ -128,6 +128,12 @@ def complete_temp_upload(temp, download_url=None):
services.ensure_visual_hashes(item) services.ensure_visual_hashes(item)
temp.file.delete(save=False) temp.file.delete(save=False)
# Warm the preview while the import is still off the request path.
try:
services.ensure_thumbnail(item)
except Exception: # noqa: BLE001 - a preview must not fail the import
logger.exception("Could not warm the thumbnail for J-%s", item.id)
temp.library_item = item temp.library_item = item
temp.status = TempUpload.STATUS_COMPLETED temp.status = TempUpload.STATUS_COMPLETED