Make thumbnail generation atomic and warm it on import
- write thumbnails to a .part file and os.replace() them, so concurrent requests never read a half-written JPEG - a stale thumbnail plus a vanished source no longer raises through the request (getmtime on a missing file returned 500); it falls back cleanly - ensure_thumbnail(item) warms the preview when a file is indexed, keeping image decoding out of the request path
This commit is contained in:
@@ -6,6 +6,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
@@ -481,15 +482,38 @@ def sanitize_iqdb_results(results):
|
|||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
|
|
||||||
|
def _thumbnail_is_fresh(target, path):
|
||||||
|
"""True when the cached thumbnail exists and is at least as new as source."""
|
||||||
|
try:
|
||||||
|
stat = target.stat()
|
||||||
|
if stat.st_size <= 0:
|
||||||
|
return False
|
||||||
|
return stat.st_mtime >= os.path.getmtime(path)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _thumbs_dir():
|
||||||
|
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
|
||||||
|
thumbs_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
return thumbs_dir
|
||||||
|
|
||||||
|
|
||||||
def generate_video_thumbnail(md5, path):
|
def generate_video_thumbnail(md5, path):
|
||||||
"""Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
|
"""Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
|
||||||
if not shutil.which("ffmpeg"):
|
if not shutil.which("ffmpeg"):
|
||||||
return None
|
return None
|
||||||
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
|
try:
|
||||||
thumbs_dir.mkdir(parents=True, exist_ok=True)
|
thumbs_dir = _thumbs_dir()
|
||||||
|
except OSError:
|
||||||
|
logger.exception("Could not create the thumbnail folder")
|
||||||
|
return None
|
||||||
target = thumbs_dir / f"{md5}.jpg"
|
target = thumbs_dir / f"{md5}.jpg"
|
||||||
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
|
if _thumbnail_is_fresh(target, path):
|
||||||
return target
|
return target
|
||||||
|
# Write beside the target and move it into place, so a concurrent request
|
||||||
|
# can never read a half-written JPEG.
|
||||||
|
temp = thumbs_dir / f".{md5}.{uuid.uuid4().hex}.part.jpg"
|
||||||
command = [
|
command = [
|
||||||
"ffmpeg",
|
"ffmpeg",
|
||||||
"-y",
|
"-y",
|
||||||
@@ -503,11 +527,14 @@ def generate_video_thumbnail(md5, path):
|
|||||||
"scale=480:-2",
|
"scale=480:-2",
|
||||||
"-loglevel",
|
"-loglevel",
|
||||||
"error",
|
"error",
|
||||||
str(target),
|
str(temp),
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
subprocess.run(command, check=True, capture_output=True, timeout=60)
|
subprocess.run(command, check=True, capture_output=True, timeout=60)
|
||||||
|
os.replace(temp, target)
|
||||||
except (subprocess.SubprocessError, OSError):
|
except (subprocess.SubprocessError, OSError):
|
||||||
|
logger.exception("Could not build a video thumbnail for %s", path)
|
||||||
|
temp.unlink(missing_ok=True)
|
||||||
return None
|
return None
|
||||||
return target if target.exists() else None
|
return target if target.exists() else None
|
||||||
|
|
||||||
@@ -517,14 +544,18 @@ def generate_image_thumbnail(md5, path):
|
|||||||
|
|
||||||
The thumbnail action used to serve full-size originals for images; a
|
The thumbnail action used to serve full-size originals for images; a
|
||||||
cached 480px JPEG keeps the library grid light without touching the
|
cached 480px JPEG keeps the library grid light without touching the
|
||||||
original file. Returns ``None`` when Pillow cannot decode the format, so
|
original file. Returns ``None`` when the source is missing or Pillow
|
||||||
callers can fall back to the original.
|
cannot decode it, so callers can fall back to the original.
|
||||||
"""
|
"""
|
||||||
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
|
try:
|
||||||
thumbs_dir.mkdir(parents=True, exist_ok=True)
|
thumbs_dir = _thumbs_dir()
|
||||||
|
except OSError:
|
||||||
|
logger.exception("Could not create the thumbnail folder")
|
||||||
|
return None
|
||||||
target = thumbs_dir / f"{md5}.jpg"
|
target = thumbs_dir / f"{md5}.jpg"
|
||||||
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
|
if _thumbnail_is_fresh(target, path):
|
||||||
return target
|
return target
|
||||||
|
temp = thumbs_dir / f".{md5}.{uuid.uuid4().hex}.part.jpg"
|
||||||
try:
|
try:
|
||||||
with Image.open(path) as image:
|
with Image.open(path) as image:
|
||||||
# Animated formats: the first frame is the preview.
|
# Animated formats: the first frame is the preview.
|
||||||
@@ -532,9 +563,27 @@ def generate_image_thumbnail(md5, path):
|
|||||||
frame = ImageOps.exif_transpose(image) or image
|
frame = ImageOps.exif_transpose(image) or image
|
||||||
frame = frame.convert("RGB")
|
frame = frame.convert("RGB")
|
||||||
frame.thumbnail((480, 480))
|
frame.thumbnail((480, 480))
|
||||||
frame.save(target, "JPEG", quality=82, optimize=True)
|
frame.save(temp, "JPEG", quality=82, optimize=True)
|
||||||
|
os.replace(temp, target)
|
||||||
except Exception: # noqa: BLE001 - previews must never break serving
|
except Exception: # noqa: BLE001 - previews must never break serving
|
||||||
logger.exception("Could not build an image thumbnail for %s", path)
|
logger.exception("Could not build an image thumbnail for %s", path)
|
||||||
target.unlink(missing_ok=True)
|
temp.unlink(missing_ok=True)
|
||||||
return None
|
return None
|
||||||
return target if target.exists() else None
|
return target if target.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_thumbnail(item):
|
||||||
|
"""Generate an item's cached thumbnail if it is missing or stale.
|
||||||
|
|
||||||
|
Warming thumbnails when a file is indexed keeps image decoding out of the
|
||||||
|
request path, where the upload pipeline's hashing used to starve it.
|
||||||
|
"""
|
||||||
|
location = item.locations.first()
|
||||||
|
if location is None:
|
||||||
|
return None
|
||||||
|
path = Path(location.path)
|
||||||
|
if not path.is_file():
|
||||||
|
return None
|
||||||
|
if path.suffix.lower() in VIDEO_EXTENSIONS:
|
||||||
|
return generate_video_thumbnail(item.md5, path)
|
||||||
|
return generate_image_thumbnail(item.md5, path)
|
||||||
|
|||||||
@@ -120,6 +120,27 @@ class MediaCacheTests(TestCase):
|
|||||||
self.client.get(url, self.signed("thumbnail"))
|
self.client.get(url, self.signed("thumbnail"))
|
||||||
self.assertEqual(thumb.stat().st_mtime_ns, before)
|
self.assertEqual(thumb.stat().st_mtime_ns, before)
|
||||||
|
|
||||||
|
def test_ensure_thumbnail_reuses_the_cache(self):
|
||||||
|
first = services.ensure_thumbnail(self.item)
|
||||||
|
self.assertIsNotNone(first)
|
||||||
|
self.assertTrue(first.exists())
|
||||||
|
mtime = first.stat().st_mtime_ns
|
||||||
|
second = services.ensure_thumbnail(self.item)
|
||||||
|
self.assertEqual(second, first)
|
||||||
|
self.assertEqual(second.stat().st_mtime_ns, mtime)
|
||||||
|
|
||||||
|
def test_thumbnail_of_a_missing_source_does_not_error(self):
|
||||||
|
"""Regression: getmtime() on a vanished source used to raise a 500."""
|
||||||
|
thumbs = self._media / "thumbs"
|
||||||
|
thumbs.mkdir(parents=True, exist_ok=True)
|
||||||
|
(thumbs / f"{self.item.md5}.jpg").write_bytes(b"stale")
|
||||||
|
Path(self.item.locations.first().path).unlink()
|
||||||
|
self.assertIsNone(services.ensure_thumbnail(self.item))
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/files/J-{self.item.id}/thumbnail/", self.signed("thumbnail")
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 404)
|
||||||
|
|
||||||
def test_staged_files_cache_briefly(self):
|
def test_staged_files_cache_briefly(self):
|
||||||
temp = TempUpload.objects.create(
|
temp = TempUpload.objects.create(
|
||||||
user=self.user,
|
user=self.user,
|
||||||
|
|||||||
@@ -128,6 +128,12 @@ def complete_temp_upload(temp, download_url=None):
|
|||||||
services.ensure_visual_hashes(item)
|
services.ensure_visual_hashes(item)
|
||||||
temp.file.delete(save=False)
|
temp.file.delete(save=False)
|
||||||
|
|
||||||
|
# Warm the preview while the import is still off the request path.
|
||||||
|
try:
|
||||||
|
services.ensure_thumbnail(item)
|
||||||
|
except Exception: # noqa: BLE001 - a preview must not fail the import
|
||||||
|
logger.exception("Could not warm the thumbnail for J-%s", item.id)
|
||||||
|
|
||||||
temp.library_item = item
|
temp.library_item = item
|
||||||
temp.status = TempUpload.STATUS_COMPLETED
|
temp.status = TempUpload.STATUS_COMPLETED
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user