Compare commits

...
4 Commits
Author SHA1 Message Date
JakeBreath af378e7d71 Update the roadmap for cacheable media and session-only completions
CI / Frontend build & lint (push) Successful in 25s
CI / Backend tests (push) Successful in 2m29s
2026-09-23 18:29:49 -05:00
JakeBreath 59f397a96c Show indexed uploads as session notifications, add a Failed column
The board rendered every persisted completed row with a dismiss button and a
dismiss-all that sent the whole column to the 1000-id bulk endpoint. Now that
the backend deletes completed rows and reports them through the status feed:

- the Auto-uploaded & Indexed column renders the live feed only; a reload or
  leaving the page forgets them, and there is nothing to dismiss (just a
  client-side clear)
- duplicates complete during staging and get their card immediately from the
  upload response
- failures get their own persisted column with per-card retry and discard
- bulk discard chunks requests at 500 ids, so backlogs over the server's
  1000-id cap are still removable in one action
2026-09-23 18:28:35 -05:00
JakeBreath 9ababb8b48 Stop storing completed uploads; announce them through a live feed
Every auto-matched, duplicate or manually resolved upload left a completed
TempUpload row on the board until it was dismissed by hand, so the rows
accumulated without bound and the bulk dismiss (capped at 1000 ids) failed
once there were more. The original app never stored these: they are
notifications, not records.

- complete_temp_upload now appends {filename, J-ID, resolution, post} to a
  bounded recent_completions feed on UploadRun and deletes the staged row
- staging duplicates never create a board record either; the create response
  carries the J-ID and preview so the SPA can show the card immediately
- status_payload returns the feed (newest first, signed thumbnails) for the
  live board; finalize_round counts deleted matches in processed
- resolve/link-bulk return synthetic completion payloads
- migration 0012 adds the field and purges the existing completed backlog
  (and any stray staged files) on deploy
2026-09-23 18:24:38 -05:00
JakeBreath 37085c5dac Make media URLs stable and cacheable, add real image thumbnails
Signed media URLs embedded the current second (TimestampSigner), so every
API response re-minted every raw/thumbnail/staged URL and the browser
re-downloaded each file on every poll or navigation. Responses also carried
no cache headers at all.

- sign with a plain Signer plus a bucket-quantized exp (7d TTL, 24h bucket),
  so a URL is byte-identical across responses and rotates once a day; legacy
  TimestampSigner URLs stay accepted for one release
- add a v=<md5> version parameter to library media URLs so replacing a file
  under the same J-ID (the optimize flow) busts caches exactly when needed
- serve_file now sends ETag/Last-Modified and a private Cache-Control and
  answers conditional requests with 304; library media gets max-age 6d +
  immutable, staged/similarity files 1h
- build cached 480px JPEG thumbnails for images (Pillow, keyed by MD5 under
  MEDIA_ROOT/thumbs) instead of serving full-size originals through the
  thumbnail endpoint; the library grid uses thumbnail_url for images too
2026-09-23 18:13:52 -05:00
16 changed files with 836 additions and 176 deletions
+14 -3
View File
@@ -20,6 +20,13 @@ they land.
- [x] Tag cloud in the sidebar (click to search, hidden from guests for
blacklisted items)
- [x] Status filter (matched / not_found / deleted / custom / unknown)
- [x] **Cacheable media serving**
- [x] Stable signed URLs (7 d TTL, 24 h rotation) plus a `v=<md5>` cache
buster, so replacing a file under the same J-ID invalidates it
- [x] ETag/Last-Modified and a private `Cache-Control` (immutable for
versioned media), conditional 304s
- [x] Real 480 px image thumbnails (cached by MD5) instead of serving
full-size originals through the thumbnail endpoint
- [x] **Random image** endpoint and page: `GET /api/random/` (aliases
`/random`, `/random/`) with `rating=s,q,e` filters; fastfetch mode
@@ -104,12 +111,16 @@ Files now stage first and are resolved before entering the library.
shell indicator; rate-limited runs retry with backoff
- [x] Perceptual-hash comparison against the library, loaded once per batch
- [x] IQDB candidates stored with one batched enrichment request
- [x] Indexed uploads are announced through a bounded per-run feed and the
staged row is deleted; nothing persists on the board to dismiss
- [x] **Upload UI**
- [x] Three-column board: Pending & Unmatched / Visual Similarity Detected /
Auto-uploaded & Indexed, private per user (staff included)
- [x] Four-column board: Pending & Unmatched / Visual Similarity Detected /
Auto-uploaded & Indexed (session feed) / Failed, private per user
(staff included)
- [x] Metadata modal (link to e621 post, IQDB candidates, custom metadata)
- [x] Per-file progress plus background pipeline status
- [x] Bulk actions: bulk rate, discard all (pending/visual), dismiss all
- [x] Bulk actions: bulk rate and discard all (chunked past the API's
1000-id cap)
## 4. Staff tools
+35 -21
View File
@@ -36,6 +36,7 @@ from apps.library.models import (
TempUpload,
)
from apps.library.services import MEDIA_FILE_SALT
from apps.library.signing_urls import sign_payload
User = get_user_model()
@@ -122,21 +123,15 @@ class SecurityTestCase(TestCase):
return item
def old_signature(self, item, action="raw", age=3 * 86400):
"""A valid signature minted `age` seconds ago."""
real_time = signing.time
class Backdated:
def time(self):
return real_time.time() - age
try:
signing.time = Backdated()
return signing.dumps(
{"item": item.id, "user": self.users["sec-uploader"].id, "action": action},
salt=MEDIA_FILE_SALT,
"""A signed media URL whose expiry is `age` seconds in the past."""
return signing.Signer(salt=MEDIA_FILE_SALT).sign_object(
{
"item": item.id,
"user": self.users["sec-uploader"].id,
"action": action,
"exp": int(time.time()) - age,
}
)
finally:
signing.time = real_time
class GuestVisibilityTests(SecurityTestCase):
@@ -183,9 +178,9 @@ class GuestVisibilityTests(SecurityTestCase):
def test_authenticated_users_and_signed_urls_see_protected_items(self):
uploader = self.client_for("sec-uploader")
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
signed = signing.dumps(
signed = sign_payload(
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
salt=MEDIA_FILE_SALT,
MEDIA_FILE_SALT,
)
self.assertEqual(
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
@@ -193,9 +188,9 @@ class GuestVisibilityTests(SecurityTestCase):
)
def test_signature_integrity(self):
signed = signing.dumps(
signed = sign_payload(
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
salt=MEDIA_FILE_SALT,
MEDIA_FILE_SALT,
)
raw = f"/api/files/J-{self.hidden.id}/raw/"
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
@@ -203,10 +198,29 @@ class GuestVisibilityTests(SecurityTestCase):
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
# Valid signature, wrong action.
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
# Expired signature (minted three days ago).
# Expired signature (expiry three days ago).
expired = self.old_signature(self.hidden)
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
def test_signed_media_urls_are_stable_and_versioned(self):
"""The same item must keep the same URL across responses.
A per-second signature made browsers re-download every image on every
poll; the MD5 version parameter busts caches only when the file itself
changes (the optimize flow rewrites files under the same J-ID).
"""
item = self.visible
first = services.signed_media_url(item, self.users["sec-uploader"])
time.sleep(1.1)
second = services.signed_media_url(item, self.users["sec-uploader"])
self.assertEqual(first, second)
self.assertIn(f"v={item.md5}", first)
MediaItem.objects.filter(pk=item.pk).update(md5="b" * 32)
item.refresh_from_db()
self.assertNotEqual(
services.signed_media_url(item, self.users["sec-uploader"]), first
)
class RoleBoundaryTests(SecurityTestCase):
def test_non_uploader_is_read_only(self):
@@ -451,9 +465,9 @@ class ThrottleTests(SecurityTestCase):
md5=hashlib.md5(b"throttle-temp").hexdigest(),
size=6,
)
signature = signing.dumps(
signature = sign_payload(
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
salt=services.UPLOAD_FILE_SALT,
services.UPLOAD_FILE_SALT,
)
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
codes = {self.guest.get(url).status_code for _ in range(150)}
@@ -0,0 +1,34 @@
# Generated by Django 6.1.1 on 2026-09-23
from django.db import migrations, models
def purge_completed_uploads(apps, schema_editor):
"""Completed staged uploads are notifications, not records.
The board used to keep every auto-uploaded/indexed row until it was
dismissed by hand, so they accumulated without bound (and bulk dismissal
is capped at 1000 ids). Completions now live in a small per-run feed, so
the old rows are removed here.
"""
TempUpload = apps.get_model("library", "TempUpload")
for temp in TempUpload.objects.filter(status="completed").iterator():
if temp.file:
temp.file.delete(save=False)
temp.delete()
class Migration(migrations.Migration):
dependencies = [
("library", "0011_upload_pipeline"),
]
operations = [
migrations.AddField(
model_name="uploadrun",
name="recent_completions",
field=models.JSONField(blank=True, default=list),
),
migrations.RunPython(purge_completed_uploads, migrations.RunPython.noop),
]
+5
View File
@@ -229,6 +229,11 @@ class UploadRun(models.Model):
matched = models.IntegerField(default=0)
failed = models.IntegerField(default=0)
error = models.TextField(blank=True, default="")
# Rolling feed of recent completions for the upload board. Completed
# staged uploads are deleted as soon as they are indexed; this only tells
# the live page "filename -> J-x" while it watches. Bounded, never
# dismissed, and ignored by fresh page loads.
recent_completions = models.JSONField(default=list, blank=True)
started_at = models.DateTimeField(null=True, blank=True)
updated_at = models.DateTimeField(auto_now=True)
+5 -5
View File
@@ -3,7 +3,6 @@ from datetime import timedelta
from pathlib import Path
from django.conf import settings
from django.core import signing
from rest_framework import serializers
from .models import (
@@ -20,6 +19,7 @@ from .services import (
VIDEO_EXTENSIONS,
signed_media_url,
)
from .signing_urls import sign_payload
class MediaLocationSerializer(serializers.ModelSerializer):
@@ -199,9 +199,9 @@ class TempUploadSerializer(serializers.ModelSerializer):
user = self._request_user()
if user is None:
return None
signature = signing.dumps(
signature = sign_payload(
{"temp": str(obj.id), "user": user.id},
salt=UPLOAD_FILE_SALT,
UPLOAD_FILE_SALT,
)
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
request = self.context.get("request")
@@ -339,9 +339,9 @@ class SimilarityCheckSerializer(serializers.ModelSerializer):
user = self._request_user()
if user is None or not obj.file:
return None
signature = signing.dumps(
signature = sign_payload(
{"check": str(obj.id), "user": user.id},
salt=UPLOAD_FILE_SALT,
UPLOAD_FILE_SALT,
)
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
request = self.context.get("request")
+84 -11
View File
@@ -6,16 +6,20 @@ import os
import re
import shutil
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
import imagehash
from django.conf import settings
from django.core import signing
from django.http import FileResponse, Http404, HttpResponse
from django.utils.cache import get_conditional_response
from django.utils.http import http_date
from django.utils.text import get_valid_filename
from PIL import Image
from PIL import Image, ImageOps
from .models import MediaItem, MediaLocation
from .signing_urls import sign_payload
logger = logging.getLogger(__name__)
@@ -37,6 +41,11 @@ UPLOAD_FILE_SALT = "j621.upload-file"
MEDIA_FILE_SALT = "j621.media-file"
CHUNK_SIZE = 1024 * 1024
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
# Versioned media URLs are immutable, so they may sit in the browser cache for
# as long as the signature is guaranteed to stay valid (7 days).
MEDIA_CACHE_SECONDS = 6 * 86400
# Staged uploads and similarity files can be deleted at any moment.
TEMP_CACHE_SECONDS = 3600
def compute_md5(path):
@@ -78,14 +87,24 @@ def signed_media_url(item, user, action="raw", request=None):
With a ``request`` the URL is absolute, so the SPA also works when it is
served from a different origin; without one it stays relative.
The ``v`` parameter is the item's MD5: it busts the browser cache exactly
when the file is replaced (the optimize flow rewrites files under the same
J-ID), which is what lets the URL be cached for days instead of re-minted
on every response.
"""
path = f"/api/files/J-{item.id}/{action}/"
params = {}
if user is not None and getattr(user, "is_authenticated", False):
signature = signing.dumps(
params = {
"v": item.md5,
"sig": sign_payload(
{"item": item.id, "user": user.id, "action": action},
salt=MEDIA_FILE_SALT,
)
path = f"{path}?sig={signature}"
MEDIA_FILE_SALT,
),
}
if params:
path = f"{path}?{urlencode(params)}"
if request is None:
return path
return request.build_absolute_uri(path)
@@ -207,12 +226,36 @@ class RangeFileWrapper:
self.file.close()
def serve_file(request, path, download=False):
"""Serve a file with HTTP range support (needed for video seeking)."""
def _apply_cache_headers(response, cache_control, etag, mtime):
response["Cache-Control"] = cache_control
response["ETag"] = etag
response["Last-Modified"] = http_date(mtime)
return response
def serve_file(request, path, download=False, *, max_age=TEMP_CACHE_SECONDS, immutable=False):
"""Serve a file with HTTP range support (needed for video seeking).
Responses carry validators (ETag/Last-Modified) and a private
``Cache-Control`` so browsers reuse media instead of re-downloading it on
every SPA poll. ``max_age``/``immutable`` are chosen by the caller: versioned
library media can be cached hard, staged files only briefly.
"""
path = Path(path)
if not path.is_file():
raise Http404
size = path.stat().st_size
stat = path.stat()
size = stat.st_size
etag = f'W/"{size:x}-{stat.st_mtime_ns:x}"'
last_modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
conditional = get_conditional_response(
request, etag=etag, last_modified=last_modified
)
if conditional is not None:
return conditional
cache_control = f"private, max-age={int(max_age)}"
if immutable:
cache_control += ", immutable"
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
range_header = request.headers.get("Range", "").strip()
if range_header:
@@ -240,7 +283,9 @@ def serve_file(request, path, download=False):
response["Content-Length"] = str(length)
response["Content-Range"] = f"bytes {start}-{end}/{size}"
response["Accept-Ranges"] = "bytes"
return response
return _apply_cache_headers(
response, cache_control, etag, stat.st_mtime
)
response = FileResponse(
open(path, "rb"),
content_type=content_type,
@@ -248,7 +293,7 @@ def serve_file(request, path, download=False):
filename=path.name,
)
response["Accept-Ranges"] = "bytes"
return response
return _apply_cache_headers(response, cache_control, etag, stat.st_mtime)
class DownloadCancelled(Exception):
@@ -465,3 +510,31 @@ def generate_video_thumbnail(md5, path):
except (subprocess.SubprocessError, OSError):
return None
return target if target.exists() else None
def generate_image_thumbnail(md5, path):
"""Downscale an image, cached under MEDIA_ROOT/thumbs like video thumbs.
The thumbnail action used to serve full-size originals for images; a
cached 480px JPEG keeps the library grid light without touching the
original file. Returns ``None`` when Pillow cannot decode the format, so
callers can fall back to the original.
"""
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
thumbs_dir.mkdir(parents=True, exist_ok=True)
target = thumbs_dir / f"{md5}.jpg"
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
return target
try:
with Image.open(path) as image:
# Animated formats: the first frame is the preview.
image.seek(0)
frame = ImageOps.exif_transpose(image) or image
frame = frame.convert("RGB")
frame.thumbnail((480, 480))
frame.save(target, "JPEG", quality=82, optimize=True)
except Exception: # noqa: BLE001 - previews must never break serving
logger.exception("Could not build an image thumbnail for %s", path)
target.unlink(missing_ok=True)
return None
return target if target.exists() else None
+57
View File
@@ -0,0 +1,57 @@
"""Stable, expiring signatures for media URLs.
The SPA loads media with ``<img>``/``<video>`` tags, which cannot send the
API's ``Authorization`` header, so those URLs carry a signature instead. The
signature has to be *stable*: a URL that changes on every response makes the
browser treat every refetch as a new resource and re-download the file.
URLs are signed with a plain ``Signer`` (no per-second timestamp) plus an
explicit ``exp`` claim quantized to a bucket, so every request inside a bucket
mints the exact same URL. The URL rotates once per bucket and is valid for at
least ``URL_TTL_SECONDS`` and at most ``URL_TTL_SECONDS + URL_BUCKET_SECONDS``.
"""
import time
from django.core import signing
URL_TTL_SECONDS = 7 * 86400
URL_BUCKET_SECONDS = 24 * 3600
_BUCKETS = URL_TTL_SECONDS // URL_BUCKET_SECONDS
def _expiry(now=None):
current = time.time() if now is None else now
bucket = int(current // URL_BUCKET_SECONDS)
return (bucket + _BUCKETS + 1) * URL_BUCKET_SECONDS
def sign_payload(payload, salt, now=None):
"""Sign a payload with a stable, bucket-quantized expiry."""
return signing.Signer(salt=salt).sign_object(
{**payload, "exp": _expiry(now)}
)
def load_payload(signature, salt, legacy_max_age=86400):
"""Verify a signed payload; ``None`` when missing, tampered with or expired.
Signatures minted before the stable scheme (``TimestampSigner``) are still
accepted for one release so pages open across the deploy keep working.
"""
try:
data = signing.Signer(salt=salt).unsign_object(signature)
except signing.BadSignature:
try:
return signing.TimestampSigner(salt=salt).unsign_object(
signature, max_age=legacy_max_age
)
except signing.BadSignature:
return None
if not isinstance(data, dict):
return None
try:
expired = int(data.get("exp", 0)) < time.time()
except (TypeError, ValueError):
return None
return None if expired else data
+2 -9
View File
@@ -11,7 +11,6 @@ from datetime import timedelta
from pathlib import Path
from django.conf import settings
from django.core import signing
from django.utils import timezone
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import action
@@ -22,6 +21,7 @@ from rest_framework.response import Response
from . import services
from .models import MediaItem, SimilarityCheck
from .serializers import SimilarityCheckSerializer
from .signing_urls import load_payload
from .tools import item_brief
from .uploads import find_library_matches
@@ -149,14 +149,7 @@ class SimilarityCheckViewSet(
check = self.get_queryset().filter(pk=pk).first()
else:
signature = request.query_params.get("sig")
payload = None
if signature:
try:
payload = signing.loads(
signature, salt=services.UPLOAD_FILE_SALT, max_age=86400
)
except signing.BadSignature:
payload = None
payload = load_payload(signature, services.UPLOAD_FILE_SALT) if signature else None
if payload and payload.get("check") == str(pk):
check = SimilarityCheck.objects.filter(pk=pk).first()
if check is None or not check.file:
@@ -0,0 +1,139 @@
"""Signed media URLs must be stable, versioned and cacheable.
Regression: signatures embedded the current second, so every API response
re-minted every URL and browsers re-downloaded each image on every poll; the
file responses also carried no cache headers at all.
"""
import base64
import hashlib
import io
import shutil
import tempfile
import time
from pathlib import Path
from django.contrib.auth import get_user_model
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import Client, TestCase, override_settings
from PIL import Image
from rest_framework.authtoken.models import Token
from apps.library import services
from apps.library.models import MediaItem, MediaLocation, TempUpload
from apps.library.signing_urls import sign_payload
User = get_user_model()
TINY_PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
)
def png_bytes(width=1200, height=800, color=(20, 120, 200)):
buffer = io.BytesIO()
Image.new("RGB", (width, height), color).save(buffer, format="PNG")
return buffer.getvalue()
class MediaCacheTests(TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls._tmp = tempfile.mkdtemp(prefix="j621-cache-")
cls._media = Path(cls._tmp) / "media"
cls._watched = cls._media / "library"
cls._watched.mkdir(parents=True, exist_ok=True)
cls._settings = override_settings(
MEDIA_ROOT=str(cls._media), WATCHED_FOLDER=str(cls._watched)
)
cls._settings.enable()
@classmethod
def tearDownClass(cls):
cls._settings.disable()
shutil.rmtree(cls._tmp, ignore_errors=True)
super().tearDownClass()
def setUp(self):
self.user = User.objects.create_user(
username="cache-uploader", password="cache-pass-123456"
)
self.user.role = "uploader"
self.user.save(update_fields=["role"])
self.token = Token.objects.create(user=self.user).key
payload = png_bytes()
path = self._watched / "cache-image.png"
path.write_bytes(payload)
self.item = MediaItem.objects.create(
md5=hashlib.md5(payload).hexdigest(), size=len(payload)
)
MediaLocation.objects.create(
item=self.item, path=str(path), rel_path=path.name, mtime=time.time()
)
self.client = Client()
def signed(self, action):
return {
"sig": sign_payload(
{"item": self.item.id, "user": self.user.id, "action": action},
services.MEDIA_FILE_SALT,
)
}
def test_media_response_carries_cache_headers(self):
response = self.client.get(
f"/api/files/J-{self.item.id}/raw/", self.signed("raw")
)
self.assertEqual(response.status_code, 200)
self.assertIn("private", response["Cache-Control"])
self.assertIn(
f"max-age={services.MEDIA_CACHE_SECONDS}", response["Cache-Control"]
)
self.assertIn("immutable", response["Cache-Control"])
self.assertTrue(response["ETag"])
self.assertTrue(response["Last-Modified"])
def test_media_revalidation_returns_304(self):
url = f"/api/files/J-{self.item.id}/raw/"
first = self.client.get(url, self.signed("raw"))
second = self.client.get(
url, self.signed("raw"), HTTP_IF_NONE_MATCH=first["ETag"]
)
self.assertEqual(second.status_code, 304)
self.assertEqual(second.content, b"")
def test_image_thumbnail_is_generated_and_reused(self):
url = f"/api/files/J-{self.item.id}/thumbnail/"
response = self.client.get(url, self.signed("thumbnail"))
self.assertEqual(response.status_code, 200)
self.assertEqual(response["Content-Type"], "image/jpeg")
thumb = self._media / "thumbs" / f"{self.item.md5}.jpg"
self.assertTrue(thumb.exists())
with Image.open(thumb) as image:
self.assertLessEqual(max(image.size), 480)
before = thumb.stat().st_mtime_ns
self.client.get(url, self.signed("thumbnail"))
self.assertEqual(thumb.stat().st_mtime_ns, before)
def test_staged_files_cache_briefly(self):
temp = TempUpload.objects.create(
user=self.user,
file=SimpleUploadedFile("staged.png", TINY_PNG, content_type="image/png"),
original_filename="staged.png",
md5=hashlib.md5(b"staged").hexdigest(),
size=len(TINY_PNG),
)
signature = sign_payload(
{"temp": str(temp.id), "user": self.user.id}, services.UPLOAD_FILE_SALT
)
response = self.client.get(
f"/api/uploads/{temp.id}/file/", {"sig": signature}
)
self.assertEqual(response.status_code, 200)
self.assertIn(
f"max-age={services.TEMP_CACHE_SECONDS}", response["Cache-Control"]
)
self.assertNotIn("immutable", response["Cache-Control"])
+81 -21
View File
@@ -170,12 +170,18 @@ class StagedUploadWorkflowTests(TestCase):
self.assertEqual(len(body["resolved"]), 2)
self.assertEqual(body["errors"], [])
for temp in (first, second):
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertIsNotNone(temp.library_item_id)
self.assertEqual(temp.library_item.rating, "q")
self.assertEqual(temp.library_item.uploaded_by_id, self.uploader.id)
# Completed uploads are notifications now: the staged rows are gone
# and the live feed carries the filename -> J-ID mapping.
self.assertFalse(
TempUpload.objects.filter(pk__in=[first.id, second.id]).exists()
)
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(
{entry["filename"] for entry in feed}, {"one.png", "two.png"}
)
for item in MediaItem.objects.all():
self.assertEqual(item.rating, "q")
self.assertEqual(item.uploaded_by_id, self.uploader.id)
untouched.refresh_from_db()
self.assertEqual(untouched.status, TempUpload.STATUS_PENDING)
@@ -231,6 +237,25 @@ class StagedUploadWorkflowTests(TestCase):
self.assertEqual(response.status_code, 200)
return seed
def test_duplicate_upload_returns_a_completion_without_a_record(self):
client = self.api_client(self.uploader)
first = self.upload_via_api(client, "same.png")
self.assertEqual(first.status_code, 201)
# Index the first upload so the second one is byte-identical.
seed = TempUpload.objects.get(pk=first.json()["temp_id"])
self.resolve_bulk(client, [seed.id], "s")
response = self.upload_via_api(client, "same.png")
self.assertEqual(response.status_code, 201)
body = response.json()
self.assertEqual(body["status"], TempUpload.STATUS_COMPLETED)
self.assertEqual(body["resolution"], TempUpload.RESOLUTION_DUPLICATE)
self.assertTrue(body["library_j_id"].startswith("J-"))
# Duplicates never become board records; the feed announces them.
self.assertFalse(TempUpload.objects.filter(pk=body["temp_id"]).exists())
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(feed[-1]["filename"], "same.png")
def test_upload_defers_visual_similarity_to_its_phase(self):
client = self.api_client(self.uploader)
self.seed_library_item(client, "seed-defer")
@@ -312,14 +337,23 @@ class StagedUploadWorkflowTests(TestCase):
body = response.json()
self.assertEqual(body["errors"], [])
self.assertEqual(len(body["updated"]), 2)
for temp, post_id in ((first, 900001), (second, 900002)):
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertEqual(temp.library_item_id is not None, True)
self.assertEqual(temp.e621_post_id, post_id)
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(temp.library_item.e621_post_id, post_id)
self.assertEqual(MediaItem.objects.count(), 2)
for entry, post_id in zip(body["updated"], (900001, 900002)):
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(entry["e621_post_id"], post_id)
self.assertTrue(entry["library_j_id"].startswith("J-"))
# Indexed uploads no longer leave a board record; the completion feed
# carries them for the live page instead.
self.assertFalse(TempUpload.objects.exists())
self.assertEqual(
{item.e621_post_id for item in MediaItem.objects.all()},
{900001, 900002},
)
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(len(feed), 2)
self.assertEqual(
{entry["filename"] for entry in feed},
{"bulk-link-1.png", "bulk-link-2.png"},
)
class IqdbRecordingTests(TestCase):
@@ -461,6 +495,7 @@ class UploadPipelineTests(TestCase):
def test_md5_match_auto_imports_the_file(self):
temp = self.stage(label="match")
temp_id = temp.id
post = {
"id": 123456,
"rating": "s",
@@ -477,17 +512,42 @@ class UploadPipelineTests(TestCase):
):
upload_pipeline.run_pipeline(self.uploader.id)
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(temp.e621_post_id, 123456)
self.assertIsNotNone(temp.library_item_id)
self.assertIsNotNone(temp.e621_checked_at)
self.assertIsNone(temp.claimed_at)
# The indexed upload leaves no board record; the feed reports it.
self.assertFalse(TempUpload.objects.filter(pk=temp_id).exists())
item = MediaItem.objects.get(e621_post_id=123456)
self.assertEqual(item.uploaded_by_id, self.uploader.id)
run = UploadRun.objects.get(user=self.uploader)
self.assertEqual(run.status, UploadRun.STATUS_IDLE)
self.assertEqual(run.matched, 1)
self.assertEqual(run.processed, 1)
self.assertEqual(len(run.recent_completions), 1)
entry = run.recent_completions[0]
self.assertEqual(entry["id"], str(temp_id))
self.assertEqual(entry["item_id"], item.id)
self.assertEqual(entry["filename"], "match.png")
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
def test_status_reports_the_completion_feed(self):
temp = self.stage(label="feed")
client = self.api_client(self.uploader)
post = {
"id": 654321,
"rating": "s",
"file": {"md5": temp.md5, "url": "https://static1.e621.net/data/f.png"},
}
with mock.patch.object(
upload_pipeline.e621,
"check_md5_batch",
return_value={temp.md5: post},
):
upload_pipeline.run_pipeline(self.uploader.id)
body = client.get("/api/uploads/status/").json()
completions = body["recent_completions"]
self.assertEqual(len(completions), 1)
self.assertEqual(completions[0]["filename"], "feed.png")
self.assertEqual(completions[0]["j_id"], f"J-{MediaItem.objects.get().id}")
self.assertIn("/thumbnail/", completions[0]["thumbnail_url"])
def test_unmatched_file_runs_every_phase(self):
temp = self.stage(label="nomatch")
+73 -14
View File
@@ -29,7 +29,7 @@ from django.db.models import F, Q
from django.utils import timezone
from . import e621, services
from .models import TempUpload, UploadRun
from .models import MediaItem, TempUpload, UploadRun
logger = logging.getLogger(__name__)
@@ -46,6 +46,9 @@ MAX_ATTEMPTS = 3
# Round-level e621 retries before the run is paused.
ROUND_ATTEMPTS = 3
ROUND_RETRY_SECONDS = 20
# Completions kept in the live feed. The board only shows what happened while
# the page was open, so a bounded rolling window is plenty.
COMPLETION_FEED_LIMIT = 200
WORK_STATUSES = (TempUpload.STATUS_PENDING, TempUpload.STATUS_VISUAL_MATCH)
VIDEO_RE = r"\.(mp4|webm)$"
@@ -116,7 +119,7 @@ def waiting_counts(user):
}
def status_payload(user):
def status_payload(user, request=None):
"""Cheap state for the shell/upload page to poll."""
run = UploadRun.objects.filter(user=user).first()
outstanding = count_outstanding(user)
@@ -141,10 +144,67 @@ def status_payload(user):
"error": run.error if run is not None else "",
"outstanding": outstanding,
"waiting": waiting_counts(user),
"recent_completions": completion_payload(run, user, request=request),
"updated_at": run.updated_at.isoformat() if run is not None else None,
}
def completion_payload(run, user, request=None):
"""The live completion feed: filename -> J-ID for freshly indexed uploads.
Completed ``TempUpload`` rows are deleted, so this is the only place the
board learns about them. It is a notification feed, not durable state:
bounded, never dismissed, and ignored by fresh page loads.
"""
entries = list(run.recent_completions or []) if run is not None else []
if not entries:
return []
ids = [entry.get("item_id") for entry in entries if entry.get("item_id")]
items = MediaItem.objects.in_bulk(ids)
out = []
for entry in reversed(entries): # newest first
item = items.get(entry.get("item_id"))
if item is None:
continue
out.append(
{
"id": entry.get("id"),
"filename": entry.get("filename"),
"j_id": f"J-{item.id}",
"resolution": entry.get("resolution", ""),
"post_id": entry.get("post_id"),
"thumbnail_url": services.signed_media_url(
item, user, "thumbnail", request=request
),
"at": entry.get("at"),
}
)
return out
def record_completion(temp, item, resolution=""):
"""Append one completion to the owner's feed; never fails an import."""
entry = {
"id": str(temp.pk),
"filename": temp.original_filename,
"item_id": item.pk,
"resolution": resolution or temp.resolution or "",
"post_id": temp.e621_post_id,
"at": timezone.now().isoformat(),
}
try:
with transaction.atomic():
run, _ = UploadRun.objects.select_for_update().get_or_create(
user_id=temp.user_id
)
feed = list(run.recent_completions or [])
feed.append(entry)
run.recent_completions = feed[-COMPLETION_FEED_LIMIT:]
run.save(update_fields=["recent_completions", "updated_at"])
except Exception: # noqa: BLE001 - a notification must not break an import
logger.exception("Could not record the completion of %s", temp.pk)
def reap_stale_claims():
"""Queue rows left claimed by a recycled worker and pause dead runs."""
cutoff = timezone.now() - STALE_CLAIM_AFTER
@@ -237,20 +297,17 @@ def run_pipeline(user_id):
break
process_round(run, user, rows)
except PipelinePaused as exc:
run.status = UploadRun.STATUS_PAUSED
run.phase = ""
run.error = str(exc)
run.save()
_save_run(run, status=UploadRun.STATUS_PAUSED, phase="", error=str(exc))
except Exception as exc: # noqa: BLE001 - surface crashes as a run error
logger.exception("Upload pipeline for user %s failed", user_id)
run.status = UploadRun.STATUS_ERROR
run.phase = ""
run.error = f"The upload pipeline stopped: {exc}"
run.save()
_save_run(
run,
status=UploadRun.STATUS_ERROR,
phase="",
error=f"The upload pipeline stopped: {exc}",
)
else:
run.status = UploadRun.STATUS_IDLE
run.phase = ""
run.save()
_save_run(run, status=UploadRun.STATUS_IDLE, phase="")
def claim_round(user, size=CLAIM_SIZE):
@@ -460,7 +517,9 @@ def finalize_round(run, ids, matched):
TempUpload.objects.filter(pk__in=release).update(claimed_at=None)
_save_run(
run,
processed=run.processed + len(finished),
# Completed rows are deleted as they are imported, so they cannot be
# seen in the refreshed rows; count the matches explicitly.
processed=run.processed + len(finished) + matched,
failed=run.failed + len(failed - finished),
matched=run.matched + matched,
)
+46 -19
View File
@@ -16,7 +16,6 @@ from urllib.parse import urlparse
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core import signing
from django.core.exceptions import ValidationError
from django.http import Http404
from django.utils import timezone
@@ -30,6 +29,7 @@ from . import services
from .models import MediaItem, TempUpload
from .permissions import CanUpload
from .serializers import TempUploadListSerializer, TempUploadSerializer
from .signing_urls import load_payload
from .tools import HASH_FIELDS, hashed_items, hashes_similarity
logger = logging.getLogger(__name__)
@@ -158,6 +158,13 @@ def complete_temp_upload(temp, download_url=None):
item.save(update_fields=update_fields + ["updated_at"])
temp.save()
from .upload_pipeline import record_completion
record_completion(temp, item)
# The board learns about completions from the live feed, so the record is
# deleted as soon as the file is indexed: nothing left to dismiss. Delete
# through the queryset so callers keep ``temp.pk`` for their response.
TempUpload.objects.filter(pk=temp.pk).delete()
return item
@@ -190,6 +197,28 @@ class TempUploadViewSet(
user=self.request.user
)
def _completed_payload(self, temp, item):
"""Synthetic row for an upload that is indexed immediately.
Duplicates and resolved uploads never leave a board record; the SPA
turns this response (or the live completion feed) into a "J-x
uploaded" card that lives only in the page session.
"""
return {
"temp_id": str(temp.pk),
"original_filename": temp.original_filename,
"md5": temp.md5,
"size": temp.size,
"status": TempUpload.STATUS_COMPLETED,
"resolution": temp.resolution,
"e621_post_id": temp.e621_post_id,
"library_j_id": f"J-{item.id}",
"file_url": None,
"preview_url": services.signed_media_url(
item, self.request.user, "thumbnail", request=self.request
),
}
def create(self, request):
upload = request.FILES.get("file")
if upload is None:
@@ -217,6 +246,13 @@ class TempUploadViewSet(
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
temp.library_item = existing
temp.file.delete(save=False)
temp.save()
from .upload_pipeline import record_completion
record_completion(temp, existing)
payload = self._completed_payload(temp, existing)
TempUpload.objects.filter(pk=temp.pk).delete()
return Response(payload, status=status.HTTP_201_CREATED)
# Visual similarity and IQDB run in the background pipeline so a large
# batch uploads at full speed and the work survives the browser.
temp.save()
@@ -257,7 +293,7 @@ class TempUploadViewSet(
"""Cheap pipeline state for the shell indicator and the upload page."""
from .upload_pipeline import status_payload
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"])
def process(self, request):
@@ -269,7 +305,7 @@ class TempUploadViewSet(
from .upload_pipeline import start_pipeline, status_payload
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=True, methods=["post"])
def retry(self, request, pk=None):
@@ -314,7 +350,7 @@ class TempUploadViewSet(
update["status"] = TempUpload.STATUS_PENDING
TempUpload.objects.filter(pk=temp.pk).update(**update)
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"], url_path="retry-all")
def retry_all(self, request):
@@ -340,7 +376,7 @@ class TempUploadViewSet(
e621_checked_at=None,
)
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"], url_path="discard-bulk")
def discard_bulk(self, request):
@@ -393,14 +429,7 @@ class TempUploadViewSet(
if user is None:
signature = request.query_params.get("sig")
if signature:
try:
payload = signing.loads(
signature,
salt=services.UPLOAD_FILE_SALT,
max_age=86400,
)
except signing.BadSignature:
payload = None
payload = load_payload(signature, services.UPLOAD_FILE_SALT)
if payload and str(payload.get("temp")) == str(pk):
user = (
get_user_model()
@@ -501,7 +530,7 @@ class TempUploadViewSet(
temp.save()
try:
complete_temp_upload(temp, download_url=download_url)
item = complete_temp_upload(temp, download_url=download_url)
except Exception as exc: # noqa: BLE001 - report completion failures
logger.exception("Could not complete staged upload %s", temp.id)
temp.status = TempUpload.STATUS_ERROR
@@ -510,8 +539,7 @@ class TempUploadViewSet(
{"detail": f"Could not finish the upload: {exc}"},
status=status.HTTP_400_BAD_REQUEST,
)
temp.refresh_from_db()
return Response(self.get_serializer(temp).data)
return Response(self._completed_payload(temp, item))
@action(detail=False, methods=["post"], url_path="link-bulk")
def link_bulk(self, request):
@@ -583,15 +611,14 @@ class TempUploadViewSet(
candidate_url = ""
temp.save()
try:
complete_temp_upload(temp, download_url=candidate_url or None)
item = complete_temp_upload(temp, download_url=candidate_url or None)
except Exception as exc: # noqa: BLE001 - report per-file failures
logger.exception("Could not complete staged upload %s", temp.id)
temp.status = TempUpload.STATUS_ERROR
temp.save(update_fields=["status", "updated_at"])
errors.append({"temp_id": temp_id, "error": str(exc)})
continue
temp.refresh_from_db()
updated.append(self.get_serializer(temp).data)
updated.append(self._completed_payload(temp, item))
return Response({"updated": updated, "errors": errors})
+24 -10
View File
@@ -5,7 +5,6 @@ from pathlib import Path
from urllib.parse import urlparse
from django.conf import settings
from django.core import signing
from django.db.models import Min, Q
from django.http import Http404, StreamingHttpResponse
from django.shortcuts import get_object_or_404
@@ -31,6 +30,7 @@ from .serializers import (
MatchTaskSerializer,
MediaItemSerializer,
)
from .signing_urls import load_payload
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
@@ -132,11 +132,8 @@ class MediaItemViewSet(
signature = request.query_params.get("sig")
if not signature:
return None
try:
payload = signing.loads(
signature, salt=services.MEDIA_FILE_SALT, max_age=86400
)
except signing.BadSignature:
payload = load_payload(signature, services.MEDIA_FILE_SALT)
if payload is None:
return None
if payload.get("action") != action_name:
return None
@@ -158,7 +155,11 @@ class MediaItemViewSet(
status=status.HTTP_404_NOT_FOUND,
)
return services.serve_file(
request, location.path, download=request.query_params.get("download") == "1"
request,
location.path,
download=request.query_params.get("download") == "1",
max_age=services.MEDIA_CACHE_SECONDS,
immutable=True,
)
@action(detail=True, methods=["get"], throttle_classes=[])
@@ -173,13 +174,26 @@ class MediaItemViewSet(
path = Path(location.path)
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
thumbnail = services.generate_video_thumbnail(item.md5, path)
if thumbnail is None:
else:
thumbnail = services.generate_image_thumbnail(item.md5, path)
if thumbnail is not None:
return services.serve_file(
request,
thumbnail,
max_age=services.MEDIA_CACHE_SECONDS,
immutable=True,
)
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
return Response(
{"detail": "Thumbnail unavailable."},
status=status.HTTP_404_NOT_FOUND,
)
return services.serve_file(request, thumbnail)
return services.serve_file(request, path)
return services.serve_file(
request,
path,
max_age=services.MEDIA_CACHE_SECONDS,
immutable=True,
)
@action(detail=False, methods=["post"], permission_classes=[AllowAny])
def lookup(self, request):
+3 -3
View File
@@ -18,9 +18,9 @@ const ratingLabels: Record<string, string> = {
};
export function MediaCard({ item }: { item: MediaItem }) {
const preview = apiUrl(
item.kind === "video" ? item.thumbnail_url : item.raw_url,
);
// The thumbnail endpoint now builds real 480px previews for images too, so
// the grid no longer pulls full-size originals.
const preview = apiUrl(item.thumbnail_url);
const rating = item.display_rating;
return (
+198 -48
View File
@@ -28,7 +28,13 @@ import {
RATING_LABELS,
} from "@/lib/e621";
import { formatBytes } from "@/lib/format";
import type { E621IqdbCandidate, Rating, TempUpload } from "@/lib/types";
import type {
E621IqdbCandidate,
Rating,
TempUpload,
UploadCompletion,
UploadStagingResult,
} from "@/lib/types";
import {
postProcessUploads,
uploadStatusQueryKey,
@@ -272,6 +278,21 @@ function TempCard({
</div>
) : null}
{temp.status === "error" ? (
<div className="mt-2 flex flex-wrap items-center gap-1.5">
<Button className="px-2 py-1 text-xs" onClick={onRetry}>
Retry
</Button>
<Button
variant="secondary"
className="px-2 py-1 text-xs"
onClick={onOpen}
>
Set metadata
</Button>
</div>
) : null}
<button
type="button"
onClick={onDismiss}
@@ -289,6 +310,58 @@ function TempCard({
);
}
/**
* A "J-x uploaded" notification.
*
* Completions are not stored on the board: the pipeline deletes the staged
* record and announces it through the status feed, so these cards only exist
* while the page is open.
*/
function CompletionCard({ completion }: { completion: UploadCompletion }) {
const preview = apiUrl(completion.thumbnail_url ?? "");
return (
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
{preview ? (
<img
src={preview}
alt={completion.filename}
loading="lazy"
className="h-full w-full object-cover"
/>
) : (
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
No preview
</div>
)}
<span className="absolute right-2 top-2 flex items-center gap-1 rounded-full bg-ctp-green/90 px-1.5 py-0.5 font-mono text-[10px] font-medium text-ctp-crust">
<CheckCircle2 className="h-2.5 w-2.5" />
indexed
</span>
</div>
<p
className="mt-2 truncate font-mono text-xs text-ctp-subtext1"
title={completion.filename}
>
{completion.filename}
</p>
<p className="mt-1 flex items-center gap-1.5 text-xs text-ctp-green">
<CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
<Link
to={`/detail/${completion.j_id}`}
className="font-mono hover:underline"
>
{completion.j_id}
</Link>
<span className="truncate text-ctp-overlay0">
{RESOLUTION_LABELS[completion.resolution] ?? completion.resolution}
</span>
</p>
</div>
);
}
/**
* Full metadata editor for one staged upload.
*
@@ -816,6 +889,10 @@ export default function UploadPage() {
const [discarding, setDiscarding] = useState(false);
const [modalId, setModalId] = useState<string | null>(null);
const [bulkOpen, setBulkOpen] = useState(false);
// Indexed uploads shown while this page is open (never persisted).
const [completions, setCompletions] = useState<UploadCompletion[]>([]);
const completionsSeeded = useRef(false);
const seenCompletions = useRef<Set<string>>(new Set());
useEffect(() => {
const urls = objectUrls.current;
@@ -836,6 +913,27 @@ export default function UploadPage() {
});
const uploads = uploadsQuery.data ?? [];
// Completions are session notifications. Seed what the server already has
// when the page opens (so a reload does not resurrect old ones) and append
// only what happens while the page is watching.
useEffect(() => {
const feed = status?.recent_completions;
if (!feed) return;
if (!completionsSeeded.current) {
completionsSeeded.current = true;
// Merge: a staging duplicate may have been added before the first poll.
seenCompletions.current = new Set([
...seenCompletions.current,
...feed.map((entry) => entry.id),
]);
return;
}
const fresh = feed.filter((entry) => !seenCompletions.current.has(entry.id));
if (fresh.length === 0) return;
for (const entry of fresh) seenCompletions.current.add(entry.id);
setCompletions((current) => [...fresh, ...current].slice(0, 200));
}, [status]);
function cacheStatus(data: unknown) {
queryClient.setQueryData(uploadStatusQueryKey, data);
}
@@ -892,9 +990,7 @@ export default function UploadPage() {
const pending = uploads.filter((temp) => temp.status === "pending");
const visual = uploads.filter((temp) => temp.status === "visual_match");
const completed = uploads.filter(
(temp) => temp.status === "completed" || temp.status === "error",
);
const failed = uploads.filter((temp) => temp.status === "error");
// Active work stays in insertion order (the grid must not reshuffle while
// you scroll it); the in-flight files get their own strip above it.
@@ -1001,10 +1097,26 @@ export default function UploadPage() {
error: undefined,
});
try {
await uploadFile(entry.file, {}, (percent) =>
reportProgress(entry.id, percent),
const result = await uploadFile<UploadStagingResult>(
entry.file,
{},
(percent) => reportProgress(entry.id, percent),
);
finishEntry(entry);
// An exact duplicate completes during staging: show its card now.
if (result?.status === "completed" && result.library_j_id) {
const completion: UploadCompletion = {
id: result.temp_id,
filename: result.original_filename,
j_id: result.library_j_id,
resolution: result.resolution,
post_id: result.e621_post_id ?? null,
thumbnail_url: result.preview_url ?? null,
at: new Date().toISOString(),
};
seenCompletions.current.add(completion.id);
setCompletions((current) => [completion, ...current].slice(0, 200));
}
// Processing starts while the rest of the drop is still uploading.
kickPipeline();
} catch (error) {
@@ -1060,39 +1172,40 @@ export default function UploadPage() {
.catch((error) => toast.error(errorMessage(error)));
}
async function discardAll(
items: TempUpload[],
kind: "discard" | "dismiss",
) {
async function discardAll(items: TempUpload[]) {
if (items.length === 0 || discarding) return;
const noun = kind === "discard" ? "Discard" : "Dismiss";
const confirmed = await confirmAction({
title: `${noun} ${items.length} record${items.length === 1 ? "" : "s"}?`,
title: `Discard ${items.length} record${items.length === 1 ? "" : "s"}?`,
description:
kind === "discard"
? "The staged files are deleted from the staging folder and never enter the library. Files the pipeline is importing right now are skipped."
: "This only removes the board records; library files are untouched.",
confirmLabel: `${noun} all`,
danger: kind === "discard",
"The staged files are deleted from the staging folder and never enter the library. Files the pipeline is importing right now are skipped.",
confirmLabel: "Discard all",
danger: true,
});
if (!confirmed) return;
setDiscarding(true);
try {
// The endpoint caps one request at 1000 ids; large staging backlogs
// must still be discardable in one go.
const ids = items.map((temp) => temp.temp_id);
const chunkSize = 500;
const discarded: string[] = [];
const errors: { temp_id: string; error: string }[] = [];
for (let start = 0; start < ids.length; start += chunkSize) {
const result = await api<{
discarded: string[];
errors: { temp_id: string; error: string }[];
}>("/api/uploads/discard-bulk/", {
method: "POST",
json: { temp_ids: items.map((temp) => temp.temp_id) },
json: { temp_ids: ids.slice(start, start + chunkSize) },
});
const verb = kind === "discard" ? "Discarded" : "Dismissed";
if (result.discarded.length > 0) {
toast.ok(`${verb} ${result.discarded.length} record(s).`);
discarded.push(...result.discarded);
errors.push(...result.errors);
}
if (result.errors.length > 0) {
toast.error(
`${result.errors.length} record(s) could not be removed.`,
);
if (discarded.length > 0) {
toast.ok(`Discarded ${discarded.length} record(s).`);
}
if (errors.length > 0) {
toast.error(`${errors.length} record(s) could not be removed.`);
}
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
void queryClient.invalidateQueries({ queryKey: ["upload-status"] });
@@ -1127,10 +1240,11 @@ export default function UploadPage() {
);
}
const column = (
const column = <T,>(
title: string,
items: TempUpload[],
items: T[],
emptyText: string,
render: (item: T) => ReactNode,
action?: ReactNode,
) => (
<section className="flex min-w-0 flex-col gap-3">
@@ -1148,17 +1262,7 @@ export default function UploadPage() {
{emptyText}
</p>
) : (
<div className="flex flex-col gap-3">
{items.map((temp) => (
<TempCard
key={temp.temp_id}
temp={temp}
onOpen={() => setModalId(temp.temp_id)}
onRetry={() => retry(temp, "iqdb")}
onDismiss={() => dismiss(temp)}
/>
))}
</div>
<div className="flex flex-col gap-3">{items.map(render)}</div>
)}
</section>
);
@@ -1387,11 +1491,20 @@ export default function UploadPage() {
</section>
) : null}
<div className="grid gap-6 lg:grid-cols-3">
<div className="grid gap-6 lg:grid-cols-4">
{column(
"Pending & Unmatched",
pending,
"Nothing waiting.",
(temp) => (
<TempCard
key={temp.temp_id}
temp={temp}
onOpen={() => setModalId(temp.temp_id)}
onRetry={() => retry(temp, "iqdb")}
onDismiss={() => dismiss(temp)}
/>
),
pending.length > 0 ? (
<span className="flex items-center gap-1.5">
<Button
@@ -1407,7 +1520,7 @@ export default function UploadPage() {
variant="danger"
className="px-2 py-0.5 text-[11px]"
disabled={discarding}
onClick={() => void discardAll(pending, "discard")}
onClick={() => void discardAll(pending)}
title="Delete every staged file in this column"
>
<Trash2 className="h-3.5 w-3.5" />
@@ -1420,12 +1533,21 @@ export default function UploadPage() {
"Visual Similarity Detected",
visual,
"No IQDB matches right now.",
(temp) => (
<TempCard
key={temp.temp_id}
temp={temp}
onOpen={() => setModalId(temp.temp_id)}
onRetry={() => retry(temp, "iqdb")}
onDismiss={() => dismiss(temp)}
/>
),
visual.length > 0 ? (
<Button
variant="danger"
className="px-2 py-0.5 text-[11px]"
disabled={discarding}
onClick={() => void discardAll(visual, "discard")}
onClick={() => void discardAll(visual)}
title="Delete every staged file in this column"
>
<Trash2 className="h-3.5 w-3.5" />
@@ -1435,20 +1557,48 @@ export default function UploadPage() {
)}
{column(
"Auto-uploaded & Indexed",
completed,
"Nothing indexed yet.",
completed.length > 0 ? (
completions,
"Nothing indexed while you were watching.",
(completion) => (
<CompletionCard key={completion.id} completion={completion} />
),
completions.length > 0 ? (
<button
type="button"
disabled={discarding}
onClick={() => void discardAll(completed, "dismiss")}
title="Dismiss every indexed record"
onClick={() => setCompletions([])}
title="Clear these notifications (nothing is stored)"
className="rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
>
dismiss all
clear
</button>
) : null,
)}
{column(
"Failed",
failed,
"No failed uploads.",
(temp) => (
<TempCard
key={temp.temp_id}
temp={temp}
onOpen={() => setModalId(temp.temp_id)}
onRetry={() => retry(temp)}
onDismiss={() => dismiss(temp)}
/>
),
failed.length > 0 ? (
<Button
variant="danger"
className="px-2 py-0.5 text-[11px]"
disabled={discarding}
onClick={() => void discardAll(failed)}
title="Delete every failed staged file in this column"
>
<Trash2 className="h-3.5 w-3.5" />
discard all
</Button>
) : null,
)}
</div>
{modalId ? (
+24
View File
@@ -388,6 +388,28 @@ export interface TempUpload {
updated_at: string;
}
/** One upload the pipeline indexed while the page was watching. */
export interface UploadCompletion {
id: string;
filename: string;
j_id: string;
resolution: string;
post_id: number | null;
thumbnail_url: string | null;
at: string;
}
/** Response of POST /api/uploads/ — a pending row or an instant completion. */
export interface UploadStagingResult {
temp_id: string;
original_filename: string;
status: "pending" | "visual_match" | "completed" | "error";
resolution: string;
library_j_id: string | null;
preview_url: string | null;
e621_post_id?: number | null;
}
/** Progress of the server-side staged-upload pipeline. */
export interface UploadStatus {
status: "idle" | "running" | "paused" | "error";
@@ -400,6 +422,8 @@ export interface UploadStatus {
error: string;
outstanding: number;
waiting: { md5: number; visual: number; iqdb: number };
/** Session notification feed: indexed uploads, newest first. */
recent_completions: UploadCompletion[];
updated_at: string | null;
}