Match local files to e621: MD5 lookups, manual links, batch scans

- MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621.
2026-09-17 13:41:08 -05:00
parent db74f7ab18
commit 09405d1a0f
16 changed files with 1084 additions and 27 deletions
@@ -0,0 +1,67 @@
+# Generated by Django 6.1.1 on 2026-09-17 18:35
+
+import django.db.models.deletion
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+def mark_existing_matches(apps, schema_editor):
+    """Items downloaded/linked before the match cache existed."""
+    MediaItem = apps.get_model("library", "MediaItem")
+    MediaItem.objects.filter(e621_post_id__isnull=False).update(
+        e621_match_status="matched",
+        e621_checked_at=models.F("updated_at"),
+    )
+
+
+def unmark_matches(apps, schema_editor):
+    MediaItem = apps.get_model("library", "MediaItem")
+    MediaItem.objects.filter(e621_post_id__isnull=False).update(
+        e621_match_status="unknown",
+        e621_checked_at=None,
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0008_mediaitem_has_custom_data_mediaitem_search_tags'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='mediaitem',
+            name='e621_checked_at',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='mediaitem',
+            name='e621_match_status',
+            field=models.CharField(choices=[('unknown', 'Not checked'), ('matched', 'Matched'), ('not_found', 'Not found on e621'), ('deleted', 'Post deleted on e621')], db_index=True, default='unknown', max_length=20),
+        ),
+        migrations.CreateModel(
+            name='MatchTask',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('scope', models.CharField(choices=[('missing', 'Unchecked items'), ('all', 'Whole library')], default='missing', max_length=20)),
+                ('status', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('complete', 'Complete'), ('error', 'Error'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
+                ('total', models.IntegerField(default=0)),
+                ('processed', models.IntegerField(default=0)),
+                ('matched', models.IntegerField(default=0)),
+                ('not_found', models.IntegerField(default=0)),
+                ('deleted', models.IntegerField(default=0)),
+                ('failed', models.IntegerField(default=0)),
+                ('error', models.TextField(blank=True, default='')),
+                ('cancelled', models.BooleanField(default=False)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='match_tasks', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-created_at'],
+            },
+        ),
+        migrations.RunPython(mark_existing_matches, unmark_matches),
+    ]