Match local files to e621: MD5 lookups, manual links, batch scans

- MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621.
2026-09-17 13:41:08 -05:00
parent db74f7ab18
commit 09405d1a0f
16 changed files with 1084 additions and 27 deletions
@@ -0,0 +1,70 @@
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand, CommandError
+
+from apps.library import e621, matching
+from apps.library.models import MatchTask, MediaItem
+
+
+class Command(BaseCommand):
+    help = (
+        "Match library items against e621 by MD5 and refresh matched metadata "
+        "(also detects deleted posts). Uses a user's stored e621 credentials."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--user",
+            help="Username whose e621 credentials to use "
+            "(defaults to the first staff account with credentials).",
+        )
+        parser.add_argument(
+            "--scope",
+            choices=[MatchTask.SCOPE_MISSING, MatchTask.SCOPE_ALL],
+            default=MatchTask.SCOPE_MISSING,
+            help="'missing' checks unchecked items only; 'all' also refreshes "
+            "already matched items (default: missing).",
+        )
+
+    def handle(self, *args, **options):
+        User = get_user_model()
+        username = options.get("user")
+        if username:
+            user = User.objects.filter(username=username).first()
+            if user is None:
+                raise CommandError(f"No such user: {username}")
+        else:
+            user = (
+                User.objects.filter(
+                    is_staff=True, e621_api_key__gt="", e621_username__gt=""
+                )
+                .order_by("id")
+                .first()
+            )
+        if not e621.credentials_configured(user):
+            raise CommandError(
+                "No e621 credentials found. Pass --user or configure them in Account."
+            )
+
+        queryset = matching.scope_queryset(options["scope"])
+        total = queryset.count()
+        counters = {"matched": 0, "not_found": 0, "deleted": 0}
+        for index, item_id in enumerate(queryset.values_list("id", flat=True), start=1):
+            item = MediaItem.objects.filter(id=item_id).first()
+            if item is None:
+                continue
+            try:
+                status = matching.check_item(item, user)
+            except e621.E621Error as exc:
+                raise CommandError(f"Aborted after {index - 1}/{total}: {exc}")
+            counters[status] = counters.get(status, 0) + 1
+            self.stdout.write(
+                f"[{index}/{total}] J-{item_id}: {status}", ending="\r"
+            )
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                "\nDone: "
+                f"{counters['matched']} matched, {counters['not_found']} not found, "
+                f"{counters['deleted']} deleted (scope: {options['scope']})."
+            )
+        )