Upload updates

2026-09-21 09:01:01 -05:00
parent 98025e9e6d
commit 474403ffe2
18 changed files with 2539 additions and 796 deletions
@@ -0,0 +1,81 @@
+# Generated by Django 6.1.1 on 2026-09-21 13:27
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+def backfill_pipeline_checks(apps, schema_editor):
+    """Mark pre-pipeline rows as already MD5/visual-checked when they were.
+
+    Rows with an e621 post id (or already completed) clearly went through the
+    MD5 phase; rows with visual matches went through the visual phase.
+    Everything else stays unset so the new pipeline picks it up once after
+    deploy — a re-check of stale pending uploads is the desired behavior.
+    """
+    TempUpload = apps.get_model("library", "TempUpload")
+    TempUpload.objects.filter(
+        models.Q(e621_post_id__isnull=False) | models.Q(status="completed")
+    ).update(e621_checked_at=models.F("updated_at"))
+    TempUpload.objects.filter(visual_matches__isnull=False).update(
+        visual_checked_at=models.F("updated_at")
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0010_similaritycheck'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tempupload',
+            name='attempts',
+            field=models.PositiveSmallIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='tempupload',
+            name='claimed_at',
+            field=models.DateTimeField(blank=True, db_index=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='tempupload',
+            name='e621_checked_at',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='tempupload',
+            name='pipeline_error',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AddField(
+            model_name='tempupload',
+            name='visual_checked_at',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.CreateModel(
+            name='UploadRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('idle', 'Idle'), ('running', 'Running'), ('paused', 'Paused'), ('error', 'Error')], default='idle', max_length=20)),
+                ('phase', models.CharField(blank=True, choices=[('', 'None'), ('md5', 'e621 MD5'), ('visual', 'Visual similarity'), ('iqdb', 'IQDB')], default='', max_length=20)),
+                ('total', models.IntegerField(default=0)),
+                ('processed', models.IntegerField(default=0)),
+                ('matched', models.IntegerField(default=0)),
+                ('failed', models.IntegerField(default=0)),
+                ('error', models.TextField(blank=True, default='')),
+                ('started_at', models.DateTimeField(blank=True, null=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='upload_run', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-updated_at'],
+            },
+        ),
+        migrations.RunPython(
+            code=backfill_pipeline_checks, reverse_code=migrations.RunPython.noop
+        ),
+    ]
+