From 09405d1a0ffc555d9cf29fb87f784b16a0619002 Mon Sep 17 00:00:00 2001 From: JakeBreath Date: Thu, 17 Sep 2026 13:41:08 -0500 Subject: [PATCH] Match local files to e621: MD5 lookups, manual links, batch scans - MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621. --- ROADMAP.md | 15 +- backend/apps/core/views.py | 10 +- backend/apps/library/downloads.py | 9 +- backend/apps/library/e621.py | 89 ++++++++ .../library/management/commands/match_e621.py | 70 ++++++ backend/apps/library/matching.py | 209 ++++++++++++++++++ ...0009_mediaitem_e621_checked_at_and_more.py | 67 ++++++ backend/apps/library/models.py | 74 +++++++ backend/apps/library/serializers.py | 28 ++- backend/apps/library/uploads.py | 10 +- backend/apps/library/urls.py | 8 +- backend/apps/library/views.py | 123 ++++++++++- .../src/features/library/LibraryDetail.tsx | 198 ++++++++++++++++- frontend/src/features/library/LibraryPage.tsx | 24 +- frontend/src/features/library/MatchPanel.tsx | 158 +++++++++++++ frontend/src/lib/types.ts | 19 ++ 16 files changed, 1084 insertions(+), 27 deletions(-) create mode 100644 backend/apps/library/e621.py create mode 100644 backend/apps/library/management/commands/match_e621.py create mode 100644 backend/apps/library/matching.py create mode 100644 backend/apps/library/migrations/0009_mediaitem_e621_checked_at_and_more.py create mode 100644 frontend/src/features/library/MatchPanel.tsx diff --git a/ROADMAP.md b/ROADMAP.md index b20d732..31f184e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -31,10 +31,14 @@ they land. - [x] The status footer's "Active Workers" now counts running download tasks - [x] **Download to client** — streams the e621 original straight to the browser (works for guests; host-restricted proxy, no library write) -- [ ] **Match local files to e621** - - [ ] Match by MD5 from the library detail (manual post ID entry) - - [ ] Batch cache status for the whole library (`matched` / `not_found` / `deleted`) - - [ ] Show match status + e621 metadata in the library (model already supports it) +- [x] **Match local files to e621** + - [x] Match by MD5 from the library detail, plus manual post ID linking + (with an MD5-mismatch warning) and unlink + - [x] Batch cache status for the whole library (`matched` / `not_found` / + `deleted`) via background scan tasks (`/api/matches/`) and + `manage.py match_e621` + - [x] Show match status + e621 metadata in the library: status badge and + match card on the detail, and `not_found` / `deleted` status filters - [ ] **IQDB reverse search** - [ ] Search from a local file (`POST /iqdb_queries.json`) - [ ] Show candidate posts to link @@ -104,4 +108,5 @@ Files now stage first and are resolved before entering the library. - Download progress, optimization jobs and the stats "Active Workers" count all want the same background-task/progress primitive — design it once. - IQDB and e621 matching depend on e621 credentials being configured; the SPA - already talks to e621 directly where possible. + talks to e621 directly for browsing, while the backend e621 client + (`apps/library/e621.py`) handles metadata matching and batch scans. diff --git a/backend/apps/core/views.py b/backend/apps/core/views.py index 19d84f7..3debcfb 100644 --- a/backend/apps/core/views.py +++ b/backend/apps/core/views.py @@ -9,15 +9,15 @@ from .system_info import get_os_info def _worker_counts(): - """Active/queued background jobs (download tasks for now).""" - from apps.library.models import DownloadTask + """Active/queued background jobs (downloads and e621 match scans).""" + from apps.library.models import DownloadTask, MatchTask active = DownloadTask.objects.filter( status=DownloadTask.STATUS_DOWNLOADING - ).count() + ).count() + MatchTask.objects.filter(status=MatchTask.STATUS_RUNNING).count() queued = DownloadTask.objects.filter( status=DownloadTask.STATUS_PENDING - ).count() + ).count() + MatchTask.objects.filter(status=MatchTask.STATUS_PENDING).count() return {"active": active, "queued": queued} @@ -35,7 +35,7 @@ class StatusView(APIView): "version": settings.APP_VERSION, "os": get_os_info(), "watched_folder": settings.WATCHED_FOLDER, - "e621_time_ms": None, # no e621 calls in the backend yet + "e621_time_ms": None, # e621 latency is measured client-side "workers": _worker_counts(), } payload["server_time_ms"] = round((time.perf_counter() - started) * 1000, 1) diff --git a/backend/apps/library/downloads.py b/backend/apps/library/downloads.py index ac1c6f9..3c9752d 100644 --- a/backend/apps/library/downloads.py +++ b/backend/apps/library/downloads.py @@ -111,7 +111,14 @@ def run_download_task(task_id): if task.e621_data: item.e621_post_id = task.post_id item.e621_data = task.e621_data - update_fields += ["e621_post_id", "e621_data"] + item.e621_match_status = MediaItem.E621_MATCHED + item.e621_checked_at = timezone.now() + update_fields += [ + "e621_post_id", + "e621_data", + "e621_match_status", + "e621_checked_at", + ] rating = (task.e621_data or {}).get("rating") if not item.rating and rating in {"s", "q", "e"}: item.rating = rating diff --git a/backend/apps/library/e621.py b/backend/apps/library/e621.py new file mode 100644 index 0000000..4a2654a --- /dev/null +++ b/backend/apps/library/e621.py @@ -0,0 +1,89 @@ +"""Minimal e621 API client for server-side matching and metadata refresh. + +The SPA talks to e621 directly for browsing; this client exists for work the +browser cannot do reliably: long batch scans, and requests tied to a library +item rather than an open page. It uses the requesting user's stored +credentials and a global throttle (e621 asks for at most two requests per +second). +""" + +import threading +import time + +import requests +from django.conf import settings + +REQUEST_INTERVAL = 0.5 # seconds between requests, per process + + +class E621Error(Exception): + """A non-404 failure while talking to e621.""" + + +class E621NotFound(E621Error): + """The requested post does not exist (HTTP 404).""" + + +_throttle_lock = threading.Lock() +_last_request_at = 0.0 + + +def credentials_configured(user): + return bool(user is not None and getattr(user, "e621_configured", False)) + + +def _wait_for_slot(): + global _last_request_at + with _throttle_lock: + delay = _last_request_at + REQUEST_INTERVAL - time.monotonic() + if delay > 0: + time.sleep(delay) + _last_request_at = time.monotonic() + + +def get(user, path, params=None, timeout=30): + """GET an e621 API path using the user's credentials. + + Raises E621NotFound for 404s and E621Error for everything else that isn't + a 2xx, so callers never see requests exceptions. + """ + if not credentials_configured(user): + raise E621Error("Configure your e621 credentials in Account first.") + base = (getattr(user, "e621_base_url", "") or "https://e621.net").rstrip("/") + _wait_for_slot() + try: + response = requests.get( + f"{base}{path}", + params=params, + auth=(user.e621_username, user.e621_api_key), + headers={"User-Agent": settings.USER_AGENT}, + timeout=timeout, + ) + except requests.RequestException as exc: + raise E621Error(f"Could not reach e621: {exc}") from exc + if response.status_code == 404: + raise E621NotFound(f"e621 returned 404 for {path}") + if response.status_code >= 400: + raise E621Error(f"e621 replied {response.status_code} for {path}") + try: + return response.json() + except ValueError as exc: + raise E621Error("e621 returned an unexpected response.") from exc + + +def find_post_by_md5(user, md5): + """The e621 post with this exact MD5, or None.""" + payload = get(user, "/posts.json", params={"tags": f"md5:{md5}", "limit": 1}) + posts = payload.get("posts") if isinstance(payload, dict) else None + if not posts: + return None + return posts[0] + + +def fetch_post(user, post_id): + """One post by id. Raises E621NotFound when the post is gone.""" + payload = get(user, f"/posts/{int(post_id)}.json") + post = payload.get("post") if isinstance(payload, dict) else None + if not isinstance(post, dict): + raise E621Error("e621 returned an unexpected post payload.") + return post diff --git a/backend/apps/library/management/commands/match_e621.py b/backend/apps/library/management/commands/match_e621.py new file mode 100644 index 0000000..1fa4eb1 --- /dev/null +++ b/backend/apps/library/management/commands/match_e621.py @@ -0,0 +1,70 @@ +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand, CommandError + +from apps.library import e621, matching +from apps.library.models import MatchTask, MediaItem + + +class Command(BaseCommand): + help = ( + "Match library items against e621 by MD5 and refresh matched metadata " + "(also detects deleted posts). Uses a user's stored e621 credentials." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--user", + help="Username whose e621 credentials to use " + "(defaults to the first staff account with credentials).", + ) + parser.add_argument( + "--scope", + choices=[MatchTask.SCOPE_MISSING, MatchTask.SCOPE_ALL], + default=MatchTask.SCOPE_MISSING, + help="'missing' checks unchecked items only; 'all' also refreshes " + "already matched items (default: missing).", + ) + + def handle(self, *args, **options): + User = get_user_model() + username = options.get("user") + if username: + user = User.objects.filter(username=username).first() + if user is None: + raise CommandError(f"No such user: {username}") + else: + user = ( + User.objects.filter( + is_staff=True, e621_api_key__gt="", e621_username__gt="" + ) + .order_by("id") + .first() + ) + if not e621.credentials_configured(user): + raise CommandError( + "No e621 credentials found. Pass --user or configure them in Account." + ) + + queryset = matching.scope_queryset(options["scope"]) + total = queryset.count() + counters = {"matched": 0, "not_found": 0, "deleted": 0} + for index, item_id in enumerate(queryset.values_list("id", flat=True), start=1): + item = MediaItem.objects.filter(id=item_id).first() + if item is None: + continue + try: + status = matching.check_item(item, user) + except e621.E621Error as exc: + raise CommandError(f"Aborted after {index - 1}/{total}: {exc}") + counters[status] = counters.get(status, 0) + 1 + self.stdout.write( + f"[{index}/{total}] J-{item_id}: {status}", ending="\r" + ) + + self.stdout.write( + self.style.SUCCESS( + "\nDone: " + f"{counters['matched']} matched, {counters['not_found']} not found, " + f"{counters['deleted']} deleted (scope: {options['scope']})." + ) + ) diff --git a/backend/apps/library/matching.py b/backend/apps/library/matching.py new file mode 100644 index 0000000..03815b6 --- /dev/null +++ b/backend/apps/library/matching.py @@ -0,0 +1,209 @@ +"""Server-side matching of library items against e621. + +MD5 lookups, manual post links, metadata refresh (which is also how deleted +posts are detected) and the background library-wide scan. +""" + +import logging +import threading +from datetime import timedelta + +from django.db import connection +from django.utils import timezone + +from . import e621 +from .models import MatchTask, MediaItem +from .services import trim_e621_post + +logger = logging.getLogger(__name__) + +STALE_AFTER = timedelta(minutes=30) +RATING_VALUES = {"s", "q", "e"} + + +# --- single-item operations ------------------------------------------------- + + +def apply_post_to_item(item, post): + """Store trimmed e621 metadata on an item and mark it matched.""" + trimmed = trim_e621_post(post) + if trimmed is None or not trimmed.get("id"): + raise e621.E621Error("e621 returned an unexpected post payload.") + item.e621_post_id = int(trimmed["id"]) + item.e621_data = trimmed + item.e621_match_status = MediaItem.E621_MATCHED + item.e621_checked_at = timezone.now() + update_fields = [ + "e621_post_id", + "e621_data", + "e621_match_status", + "e621_checked_at", + ] + rating = trimmed.get("rating") + if not item.rating and rating in RATING_VALUES: + item.rating = rating + update_fields.append("rating") + item.save(update_fields=update_fields + ["updated_at"]) + return trimmed + + +def mark_not_found(item): + item.e621_match_status = MediaItem.E621_NOT_FOUND + item.e621_checked_at = timezone.now() + item.save(update_fields=["e621_match_status", "e621_checked_at", "updated_at"]) + + +def mark_deleted(item): + item.e621_match_status = MediaItem.E621_DELETED + item.e621_checked_at = timezone.now() + item.save(update_fields=["e621_match_status", "e621_checked_at", "updated_at"]) + + +def match_item_by_md5(item, user): + """Look the item's MD5 up on e621. Returns the resulting match status.""" + post = e621.find_post_by_md5(user, item.md5) + if post is None: + mark_not_found(item) + return item.e621_match_status + apply_post_to_item(item, post) + return item.e621_match_status + + +def refresh_item(item, user): + """Re-fetch a matched post: refresh metadata or flag it as deleted.""" + if not item.e621_post_id: + return match_item_by_md5(item, user) + try: + post = e621.fetch_post(user, item.e621_post_id) + except e621.E621NotFound: + mark_deleted(item) + return item.e621_match_status + apply_post_to_item(item, post) + return item.e621_match_status + + +def link_item_to_post(item, post_id, user): + """Manually link an item to a post id. Returns True if the MD5s match.""" + post = e621.fetch_post(user, post_id) + trimmed = apply_post_to_item(item, post) + post_md5 = str((trimmed.get("file") or {}).get("md5") or "").strip().lower() + return bool(post_md5) and post_md5 == item.md5.lower() + + +def check_item(item, user): + """Best check for this item: refresh a known link, else MD5 lookup.""" + if item.e621_post_id and item.e621_match_status == MediaItem.E621_MATCHED: + return refresh_item(item, user) + return match_item_by_md5(item, user) + + +def unlink_item(item): + """Forget the e621 link so the item can be checked again.""" + item.e621_post_id = None + item.e621_data = None + item.e621_match_status = MediaItem.E621_UNKNOWN + item.e621_checked_at = None + item.save( + update_fields=[ + "e621_post_id", + "e621_data", + "e621_match_status", + "e621_checked_at", + "updated_at", + ] + ) + return item + + +# --- batch scan ------------------------------------------------------------- + + +def reap_stale_match_tasks(): + """Surface scans left hanging by a recycled worker as errors.""" + cutoff = timezone.now() - STALE_AFTER + return MatchTask.objects.filter( + status__in=[MatchTask.STATUS_PENDING, MatchTask.STATUS_RUNNING], + updated_at__lt=cutoff, + ).update( + status=MatchTask.STATUS_ERROR, + error="The worker restarted before this scan finished.", + updated_at=timezone.now(), + ) + + +def scope_queryset(scope): + queryset = MediaItem.objects.order_by("id") + if scope == MatchTask.SCOPE_ALL: + return queryset + return queryset.filter(e621_match_status=MediaItem.E621_UNKNOWN) + + +def start_match_task(task_id): + thread = threading.Thread(target=run_match_task, args=(task_id,), daemon=True) + thread.start() + + +def run_match_task(task_id): + task = MatchTask.objects.filter(id=task_id).first() + if task is None: + return + + item_ids = list(scope_queryset(task.scope).values_list("id", flat=True)) + counters = {"matched": 0, "not_found": 0, "deleted": 0} + failed = 0 + MatchTask.objects.filter(id=task_id).update( + status=MatchTask.STATUS_RUNNING, + total=len(item_ids), + processed=0, + updated_at=timezone.now(), + ) + + def save_progress(processed): + MatchTask.objects.filter(id=task_id).update( + processed=processed, + failed=failed, + updated_at=timezone.now(), + **counters, + ) + + try: + processed = 0 + for item_id in item_ids: + if MatchTask.objects.filter(id=task_id, cancelled=True).exists(): + MatchTask.objects.filter(id=task_id).update( + status=MatchTask.STATUS_CANCELLED, updated_at=timezone.now() + ) + return + item = MediaItem.objects.filter(id=item_id).first() + if item is None: + processed += 1 + continue + try: + status = check_item(item, task.user) + counters[status] = counters.get(status, 0) + 1 + except e621.E621Error as exc: + # A broken credential or an e621 outage would fail every + # remaining item; abort instead of hammering the API. + logger.warning("Match scan %s aborted: %s", task_id, exc) + failed += 1 + MatchTask.objects.filter(id=task_id).update( + status=MatchTask.STATUS_ERROR, + processed=processed, + failed=failed, + error=str(exc), + updated_at=timezone.now(), + **counters, + ) + return + processed += 1 + save_progress(processed) + MatchTask.objects.filter(id=task_id).update( + status=MatchTask.STATUS_COMPLETE, + processed=len(item_ids), + updated_at=timezone.now(), + **counters, + ) + finally: + # Background threads hold their own DB connection; release it so + # Gunicorn workers do not leak connections when threads finish. + connection.close() diff --git a/backend/apps/library/migrations/0009_mediaitem_e621_checked_at_and_more.py b/backend/apps/library/migrations/0009_mediaitem_e621_checked_at_and_more.py new file mode 100644 index 0000000..44204f3 --- /dev/null +++ b/backend/apps/library/migrations/0009_mediaitem_e621_checked_at_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 6.1.1 on 2026-09-17 18:35 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +def mark_existing_matches(apps, schema_editor): + """Items downloaded/linked before the match cache existed.""" + MediaItem = apps.get_model("library", "MediaItem") + MediaItem.objects.filter(e621_post_id__isnull=False).update( + e621_match_status="matched", + e621_checked_at=models.F("updated_at"), + ) + + +def unmark_matches(apps, schema_editor): + MediaItem = apps.get_model("library", "MediaItem") + MediaItem.objects.filter(e621_post_id__isnull=False).update( + e621_match_status="unknown", + e621_checked_at=None, + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('library', '0008_mediaitem_has_custom_data_mediaitem_search_tags'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name='mediaitem', + name='e621_checked_at', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='mediaitem', + name='e621_match_status', + field=models.CharField(choices=[('unknown', 'Not checked'), ('matched', 'Matched'), ('not_found', 'Not found on e621'), ('deleted', 'Post deleted on e621')], db_index=True, default='unknown', max_length=20), + ), + migrations.CreateModel( + name='MatchTask', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('scope', models.CharField(choices=[('missing', 'Unchecked items'), ('all', 'Whole library')], default='missing', max_length=20)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('complete', 'Complete'), ('error', 'Error'), ('cancelled', 'Cancelled')], default='pending', max_length=20)), + ('total', models.IntegerField(default=0)), + ('processed', models.IntegerField(default=0)), + ('matched', models.IntegerField(default=0)), + ('not_found', models.IntegerField(default=0)), + ('deleted', models.IntegerField(default=0)), + ('failed', models.IntegerField(default=0)), + ('error', models.TextField(blank=True, default='')), + ('cancelled', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='match_tasks', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + migrations.RunPython(mark_existing_matches, unmark_matches), + ] diff --git a/backend/apps/library/models.py b/backend/apps/library/models.py index a208f87..6176997 100644 --- a/backend/apps/library/models.py +++ b/backend/apps/library/models.py @@ -27,6 +27,17 @@ def build_search_tags(tags, e621_data): class MediaItem(models.Model): """A logical media file, identified by its MD5 fingerprint.""" + E621_UNKNOWN = "unknown" + E621_MATCHED = "matched" + E621_NOT_FOUND = "not_found" + E621_DELETED = "deleted" + E621_STATUS_CHOICES = [ + (E621_UNKNOWN, "Not checked"), + (E621_MATCHED, "Matched"), + (E621_NOT_FOUND, "Not found on e621"), + (E621_DELETED, "Post deleted on e621"), + ] + md5 = models.CharField(max_length=32, unique=True) size = models.BigIntegerField(default=0) # Custom rating: "s" (safe), "q" (questionable), "e" (explicit) or "" (unrated). @@ -44,6 +55,14 @@ class MediaItem(models.Model): hidden_from_guests = models.BooleanField(default=False, db_index=True) e621_post_id = models.IntegerField(null=True, blank=True, db_index=True) e621_data = models.JSONField(null=True, blank=True) + # Cached result of the last MD5/manual match against e621. + e621_match_status = models.CharField( + max_length=20, + choices=E621_STATUS_CHOICES, + default=E621_UNKNOWN, + db_index=True, + ) + e621_checked_at = models.DateTimeField(null=True, blank=True) # Perceptual hashes (hex strings) used by the duplicates engine. ahash = models.CharField(max_length=32, blank=True, default="", db_index=True) dhash = models.CharField(max_length=32, blank=True, default="", db_index=True) @@ -207,3 +226,58 @@ class DownloadTask(models.Model): def __str__(self): return f"Download {self.id} ({self.status})" + + +class MatchTask(models.Model): + """A background library-wide e621 match scan with progress tracking.""" + + STATUS_PENDING = "pending" + STATUS_RUNNING = "running" + STATUS_COMPLETE = "complete" + STATUS_ERROR = "error" + STATUS_CANCELLED = "cancelled" + STATUS_CHOICES = [ + (STATUS_PENDING, "Pending"), + (STATUS_RUNNING, "Running"), + (STATUS_COMPLETE, "Complete"), + (STATUS_ERROR, "Error"), + (STATUS_CANCELLED, "Cancelled"), + ] + + SCOPE_MISSING = "missing" + SCOPE_ALL = "all" + SCOPE_CHOICES = [ + (SCOPE_MISSING, "Unchecked items"), + (SCOPE_ALL, "Whole library"), + ] + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="match_tasks", + ) + scope = models.CharField( + max_length=20, choices=SCOPE_CHOICES, default=SCOPE_MISSING + ) + status = models.CharField( + max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING + ) + total = models.IntegerField(default=0) + processed = models.IntegerField(default=0) + matched = models.IntegerField(default=0) + not_found = models.IntegerField(default=0) + deleted = models.IntegerField(default=0) + failed = models.IntegerField(default=0) + error = models.TextField(blank=True, default="") + cancelled = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self): + return f"Match scan {self.id} ({self.status})" diff --git a/backend/apps/library/serializers.py b/backend/apps/library/serializers.py index 82a8cb6..60a2619 100644 --- a/backend/apps/library/serializers.py +++ b/backend/apps/library/serializers.py @@ -4,7 +4,7 @@ from pathlib import Path from django.core import signing from rest_framework import serializers -from .models import MediaItem, MediaLocation, TempUpload, DownloadTask +from .models import MediaItem, MediaLocation, TempUpload, DownloadTask, MatchTask from .services import ( MEDIA_FILE_SALT, UPLOAD_FILE_SALT, @@ -55,6 +55,8 @@ class MediaItemSerializer(serializers.ModelSerializer): "uploaded_by_id", "e621_post_id", "e621_data", + "e621_match_status", + "e621_checked_at", "locations", "created_at", "updated_at", @@ -74,6 +76,8 @@ class MediaItemSerializer(serializers.ModelSerializer): "uploaded_by_id", "e621_post_id", "e621_data", + "e621_match_status", + "e621_checked_at", "locations", "created_at", "updated_at", @@ -220,3 +224,25 @@ class DownloadTaskSerializer(serializers.ModelSerializer): def get_library_j_id(self, obj): return f"J-{obj.library_item_id}" if obj.library_item_id else None + + +class MatchTaskSerializer(serializers.ModelSerializer): + task_id = serializers.UUIDField(source="id", read_only=True) + + class Meta: + model = MatchTask + fields = [ + "task_id", + "scope", + "status", + "total", + "processed", + "matched", + "not_found", + "deleted", + "failed", + "error", + "created_at", + "updated_at", + ] + read_only_fields = fields diff --git a/backend/apps/library/uploads.py b/backend/apps/library/uploads.py index 699f155..043d458 100644 --- a/backend/apps/library/uploads.py +++ b/backend/apps/library/uploads.py @@ -18,6 +18,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.core import signing from django.http import Http404 +from django.utils import timezone from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.parsers import FormParser, JSONParser, MultiPartParser @@ -114,7 +115,14 @@ def complete_temp_upload(temp, download_url=None): if temp.e621_post_id is not None: item.e621_post_id = temp.e621_post_id item.e621_data = temp.e621_data - update_fields += ["e621_post_id", "e621_data"] + item.e621_match_status = MediaItem.E621_MATCHED + item.e621_checked_at = timezone.now() + update_fields += [ + "e621_post_id", + "e621_data", + "e621_match_status", + "e621_checked_at", + ] if temp.custom_rating in {"s", "q", "e"} and not item.rating: item.rating = temp.custom_rating update_fields.append("rating") diff --git a/backend/apps/library/urls.py b/backend/apps/library/urls.py index dc293d0..d1a35a8 100644 --- a/backend/apps/library/urls.py +++ b/backend/apps/library/urls.py @@ -11,12 +11,18 @@ from .tools import ( VisualMatchesView, ) from .uploads import TempUploadViewSet -from .views import ClientDownloadView, DownloadTaskViewSet, MediaItemViewSet +from .views import ( + ClientDownloadView, + DownloadTaskViewSet, + MatchTaskViewSet, + MediaItemViewSet, +) router = DefaultRouter() router.register("files", MediaItemViewSet, basename="file") router.register("uploads", TempUploadViewSet, basename="upload") router.register("online/downloads", DownloadTaskViewSet, basename="download") +router.register("matches", MatchTaskViewSet, basename="match") urlpatterns = [ path("", include(router.urls)), diff --git a/backend/apps/library/views.py b/backend/apps/library/views.py index c74900c..70c4101 100644 --- a/backend/apps/library/views.py +++ b/backend/apps/library/views.py @@ -15,11 +15,16 @@ from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly from rest_framework.response import Response from rest_framework.views import APIView -from . import services +from . import e621, matching, services from .downloads import reap_stale_downloads, start_download_task -from .models import DownloadTask, MediaItem +from .matching import reap_stale_match_tasks, start_match_task +from .models import DownloadTask, MatchTask, MediaItem from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly -from .serializers import DownloadTaskSerializer, MediaItemSerializer +from .serializers import ( + DownloadTaskSerializer, + MatchTaskSerializer, + MediaItemSerializer, +) LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"} MD5_RE = re.compile(r"[0-9a-fA-F]{32}") @@ -83,12 +88,15 @@ class MediaItemViewSet( if status_value == "custom": status_query |= Q(has_custom_data=True) elif status_value == "matched": - status_query |= Q( - has_custom_data=False, e621_post_id__isnull=False - ) + status_query |= Q(e621_match_status=MediaItem.E621_MATCHED) + elif status_value == "not_found": + status_query |= Q(e621_match_status=MediaItem.E621_NOT_FOUND) + elif status_value == "deleted": + status_query |= Q(e621_match_status=MediaItem.E621_DELETED) elif status_value == "unknown": status_query |= Q( - has_custom_data=False, e621_post_id__isnull=True + has_custom_data=False, + e621_match_status=MediaItem.E621_UNKNOWN, ) queryset = queryset.filter(status_query) ordering = self.request.query_params.get("ordering", "").strip() @@ -230,6 +238,64 @@ class MediaItemViewSet( } ) + def _can_match(self, request, item): + user = request.user + return bool( + user.is_superuser + or user.role == user.ROLE_STAFF + or item.uploaded_by_id == user.id + ) + + @action(detail=True, methods=["post"], permission_classes=[CanUpload]) + def match(self, request, pk=None): + """Link this item to e621: MD5 lookup, or an explicit post id.""" + item = self.get_object() + if not self._can_match(request, item): + return Response( + {"detail": "Only the uploader or staff can match this item."}, + status=status.HTTP_403_FORBIDDEN, + ) + extra = {} + post_id = request.data.get("post_id") + try: + if post_id not in (None, ""): + text = str(post_id).strip() + if not text.isdigit(): + return Response( + {"detail": "Post ID must be a number."}, + status=status.HTTP_400_BAD_REQUEST, + ) + extra["md5_match"] = matching.link_item_to_post( + item, int(text), request.user + ) + else: + matching.check_item(item, request.user) + except e621.E621NotFound: + return Response( + {"detail": "That post does not exist on e621."}, + status=status.HTTP_404_NOT_FOUND, + ) + except e621.E621Error as exc: + return Response( + {"detail": str(exc)}, status=status.HTTP_502_BAD_GATEWAY + ) + item.refresh_from_db() + data = dict(self.get_serializer(item).data) + data.update(extra) + return Response(data) + + @action(detail=True, methods=["post"], permission_classes=[CanUpload]) + def unlink(self, request, pk=None): + """Forget this item's e621 link so it can be matched again.""" + item = self.get_object() + if not self._can_match(request, item): + return Response( + {"detail": "Only the uploader or staff can unlink this item."}, + status=status.HTTP_403_FORBIDDEN, + ) + matching.unlink_item(item) + return Response(self.get_serializer(item).data) + class DownloadTaskViewSet( mixins.ListModelMixin, @@ -295,6 +361,49 @@ class DownloadTaskViewSet( return Response({"success": True}) +class MatchTaskViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Background library-wide e621 match scans.""" + + serializer_class = MatchTaskSerializer + permission_classes = [CanUpload] + http_method_names = ["get", "post", "head", "options"] + + def get_queryset(self): + reap_stale_match_tasks() + queryset = MatchTask.objects.all() + user = self.request.user + if not (user.is_staff or user.is_superuser): + queryset = queryset.filter(user=user) + return queryset + + def create(self, request): + scope = str(request.data.get("scope") or MatchTask.SCOPE_MISSING).strip() + if scope not in {MatchTask.SCOPE_MISSING, MatchTask.SCOPE_ALL}: + scope = MatchTask.SCOPE_MISSING + task = MatchTask.objects.create(user=request.user, scope=scope) + start_match_task(task.id) + return Response( + self.get_serializer(task).data, status=status.HTTP_202_ACCEPTED + ) + + @action(detail=True, methods=["post"]) + def cancel(self, request, pk=None): + task = self.get_object() + if task.status not in {MatchTask.STATUS_PENDING, MatchTask.STATUS_RUNNING}: + return Response( + {"detail": "Task is not running."}, + status=status.HTTP_400_BAD_REQUEST, + ) + MatchTask.objects.filter(id=task.id).update( + cancelled=True, updated_at=timezone.now() + ) + return Response({"success": True}) + + class ClientDownloadView(APIView): """Stream an e621 file straight to the browser (no library write).""" diff --git a/frontend/src/features/library/LibraryDetail.tsx b/frontend/src/features/library/LibraryDetail.tsx index f8fccd6..100e153 100644 --- a/frontend/src/features/library/LibraryDetail.tsx +++ b/frontend/src/features/library/LibraryDetail.tsx @@ -12,7 +12,12 @@ import { isTypingTarget } from "@/lib/dom"; import { renderDText } from "@/lib/dtext"; import { RATING_LABELS, TAG_CATEGORY_LABELS } from "@/lib/e621"; import { formatBytes, formatDate } from "@/lib/format"; -import type { FileNeighbors, MediaItem, Rating } from "@/lib/types"; +import type { + E621MatchStatus, + FileNeighbors, + MediaItem, + Rating, +} from "@/lib/types"; import { useAuth } from "@/store/auth"; import { useUi } from "@/store/ui"; @@ -45,6 +50,188 @@ const ratingLabel: Record = { e: "Explicit", }; +const matchStatusMeta: Record< + E621MatchStatus, + { label: string; className: string } +> = { + unknown: { + label: "Not checked", + className: "border-ctp-surface2 bg-ctp-surface0 text-ctp-subtext1", + }, + matched: { + label: "Matched", + className: "border-ctp-green/40 bg-ctp-green/15 text-ctp-green", + }, + not_found: { + label: "Not found on e621", + className: "border-ctp-yellow/40 bg-ctp-yellow/15 text-ctp-yellow", + }, + deleted: { + label: "Deleted on e621", + className: "border-ctp-red/40 bg-ctp-red/15 text-ctp-red", + }, +}; + +function MatchCard({ item, canEdit }: { item: MediaItem; canEdit: boolean }) { + const queryClient = useQueryClient(); + const [postId, setPostId] = useState(""); + const [notice, setNotice] = useState<{ + kind: "ok" | "warn" | "error"; + text: string; + } | null>(null); + + function refresh() { + void queryClient.invalidateQueries({ queryKey: ["file", item.j_id] }); + void queryClient.invalidateQueries({ queryKey: ["files"] }); + void queryClient.invalidateQueries({ queryKey: ["tag-cloud"] }); + } + + const checkMutation = useMutation({ + mutationFn: () => + api(`/api/files/${item.j_id}/match/`, { + method: "POST", + json: {}, + }), + onSuccess: (data) => { + refresh(); + if (data.e621_match_status === "matched") { + setNotice({ kind: "ok", text: `Matched to post #${data.e621_post_id}.` }); + } else if (data.e621_match_status === "deleted") { + setNotice({ + kind: "error", + text: "The linked post no longer exists on e621.", + }); + } else { + setNotice({ + kind: "warn", + text: "No e621 post with this file's MD5 was found.", + }); + } + }, + onError: (error) => + setNotice({ kind: "error", text: errorMessage(error) }), + }); + + const linkMutation = useMutation({ + mutationFn: (id: number) => + api(`/api/files/${item.j_id}/match/`, { + method: "POST", + json: { post_id: id }, + }), + onSuccess: (data) => { + refresh(); + setPostId(""); + setNotice( + data.md5_match + ? { + kind: "ok", + text: `Linked to post #${data.e621_post_id} — MD5 matches.`, + } + : { + kind: "warn", + text: `Linked to post #${data.e621_post_id}, but its MD5 differs from this file.`, + }, + ); + }, + onError: (error) => + setNotice({ kind: "error", text: errorMessage(error) }), + }); + + const unlinkMutation = useMutation({ + mutationFn: () => + api(`/api/files/${item.j_id}/unlink/`, { method: "POST" }), + onSuccess: () => { + refresh(); + setNotice({ kind: "ok", text: "e621 link removed." }); + }, + onError: (error) => + setNotice({ kind: "error", text: errorMessage(error) }), + }); + + const meta = matchStatusMeta[item.e621_match_status]; + + return ( +
+
+

e621 match

+ + {meta.label} + +
+ +

+ {item.e621_checked_at + ? `Last checked ${formatDate(item.e621_checked_at)}` + : "This item has not been checked against e621 yet."} +

+ + {canEdit ? ( +
+
+ + {item.e621_post_id ? ( + + ) : null} +
+
+ + setPostId(event.target.value.replace(/[^0-9]/g, "")) + } + /> + +
+
+ ) : null} + + {notice ? ( +

+ {notice.text} +

+ ) : null} +
+ ); +} + function E621MetadataCard({ item }: { item: MediaItem }) { const data = item.e621_data; if (!item.e621_post_id || !data) return null; @@ -77,6 +264,13 @@ function E621MetadataCard({ item }: { item: MediaItem }) { + {item.e621_match_status === "deleted" ? ( +

+ This post has been deleted on e621 — the metadata below is a cached + copy. +

+ ) : null} +
Rating
@@ -501,6 +695,8 @@ export function LibraryDetail({ jId }: { jId: string }) {

+ +

Locations ({item.locations.length}) diff --git a/frontend/src/features/library/LibraryPage.tsx b/frontend/src/features/library/LibraryPage.tsx index bdaf88a..a59ffc4 100644 --- a/frontend/src/features/library/LibraryPage.tsx +++ b/frontend/src/features/library/LibraryPage.tsx @@ -5,6 +5,7 @@ import { useState, useEffect, type FormEvent } from "react"; import { CollapsibleSidebar } from "@/components/CollapsibleSidebar"; import { MediaCard } from "@/components/MediaCard"; import { Button, EmptyState, Spinner, inputClass } from "@/components/ui"; +import { MatchPanel } from "@/features/library/MatchPanel"; import { api } from "@/lib/api"; import { cn } from "@/lib/cn"; import { isTypingTarget } from "@/lib/dom"; @@ -47,6 +48,16 @@ const statusFilters = [ label: "Matched", active: "border-ctp-green/40 bg-ctp-green/15 text-ctp-green", }, + { + value: "not_found", + label: "Not found", + active: "border-ctp-yellow/40 bg-ctp-yellow/15 text-ctp-yellow", + }, + { + value: "deleted", + label: "Deleted", + active: "border-ctp-red/40 bg-ctp-red/15 text-ctp-red", + }, { value: "custom", label: "Custom", @@ -323,11 +334,14 @@ export default function LibraryPage() { ? `${query.data.count} item${query.data.count === 1 ? "" : "s"}` : "…"}

- {pageCount > 1 ? ( -

- Page {page} / {pageCount} -

- ) : null} +
+ {pageCount > 1 ? ( +

+ Page {page} / {pageCount} +

+ ) : null} + {user?.can_upload ? : null} +

{!user ? ( diff --git a/frontend/src/features/library/MatchPanel.tsx b/frontend/src/features/library/MatchPanel.tsx new file mode 100644 index 0000000..7918228 --- /dev/null +++ b/frontend/src/features/library/MatchPanel.tsx @@ -0,0 +1,158 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { RefreshCw, X } from "lucide-react"; +import { useEffect, useState } from "react"; + +import { Button, Spinner } from "@/components/ui"; +import { api, errorMessage } from "@/lib/api"; +import type { MatchTask } from "@/lib/types"; + +type Scope = "missing" | "all"; + +function summary(task: MatchTask): string { + const parts = [ + `${task.matched} matched`, + `${task.not_found} not found`, + `${task.deleted} deleted`, + ]; + if (task.failed > 0) parts.push(`${task.failed} failed`); + return parts.join(" · "); +} + +export function MatchPanel() { + const queryClient = useQueryClient(); + const [taskId, setTaskId] = useState(null); + const [menuOpen, setMenuOpen] = useState(false); + + const taskQuery = useQuery({ + queryKey: ["match-task", taskId], + queryFn: () => api(`/api/matches/${taskId}/`), + enabled: Boolean(taskId), + refetchInterval: (query) => { + const status = query.state.data?.status; + return status === "pending" || status === "running" ? 1000 : false; + }, + }); + + const task = taskQuery.data; + const running = task?.status === "pending" || task?.status === "running"; + const finishedTaskId = task && !running ? task.task_id : null; + + useEffect(() => { + if (!finishedTaskId) return; + void queryClient.invalidateQueries({ queryKey: ["files"] }); + void queryClient.invalidateQueries({ queryKey: ["tag-cloud"] }); + }, [finishedTaskId, queryClient]); + + const startMutation = useMutation({ + mutationFn: (scope: Scope) => + api("/api/matches/", { method: "POST", json: { scope } }), + onSuccess: (data) => { + setTaskId(data.task_id); + setMenuOpen(false); + }, + }); + + const cancelMutation = useMutation({ + mutationFn: () => + api<{ success: boolean }>(`/api/matches/${taskId}/cancel/`, { + method: "POST", + }), + }); + + if (running && task) { + const percent = task.total + ? Math.round((task.processed * 100) / task.total) + : 0; + return ( +
+
+
+
+
+
+ + {task.processed}/{task.total} · {task.matched} matched + + +
+ ); + } + + if (task) { + return ( +
+ {task.status === "complete" ? ( + Scan complete — {summary(task)} + ) : task.status === "cancelled" ? ( + Scan cancelled. + ) : ( + + {task.error || "The scan failed."} + + )} + +
+ ); + } + + return ( +
+ + + {menuOpen ? ( +
+ + + {startMutation.isError ? ( +

+ {errorMessage(startMutation.error)} +

+ ) : null} +
+ ) : null} +
+ ); +} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index f039aa1..65be7c4 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -81,6 +81,8 @@ export interface MediaLocation { export type Rating = "" | "s" | "q" | "e"; +export type E621MatchStatus = "unknown" | "matched" | "not_found" | "deleted"; + export interface E621StoredPost { id: number | null; created_at?: string | null; @@ -126,6 +128,8 @@ export interface MediaItem { uploaded_by_id: number | null; e621_post_id: number | null; e621_data: E621StoredPost | null; + e621_match_status: E621MatchStatus; + e621_checked_at: string | null; locations: MediaLocation[]; raw_url: string; thumbnail_url: string; @@ -185,6 +189,21 @@ export interface DownloadTask { updated_at: string; } +export interface MatchTask { + task_id: string; + scope: "missing" | "all"; + status: "pending" | "running" | "complete" | "error" | "cancelled"; + total: number; + processed: number; + matched: number; + not_found: number; + deleted: number; + failed: number; + error: string; + created_at: string; + updated_at: string; +} + export interface E621IqdbCandidate { post_id: number | null; score: number | null;