Upload pipeline: staging, MD5 auto-match, IQDB, three-column board

Backend:
- TempUpload model: staged files (pending / visual_match / completed /
  error) with resolution, e621 payload, custom metadata and IQDB data
- Files land in a temp folder and only move into the watched library
  folder once resolved; duplicates resolve immediately without a copy
- Endpoints: stage (multipart), list, retrieve, temp file, IQDB save,
  resolve (link to post or custom metadata), discard/dismiss
- cleanup_temp_uploads command for old staged files
- Replaces the old direct-to-library upload endpoint

Frontend:
- Upload page is now a three-column board (Pending & Unmatched /
  Visual Similarity Detected / Auto-uploaded & Indexed)
- After upload: MD5s are batch-checked against e621 and matches
  auto-complete with full post metadata; remaining files run through
  IQDB and move to the similarity column when candidates exist
- Metadata modal with IQDB candidates, post-ID linking and custom
  tags/rating/notes; discard and dismiss actions
- e621 client gains fetchPostsByMd5 and iqdbSearch helpers

Roadmap updated with the completed upload items.

backend/apps/library/management/commands/cleanup_temp_uploads.py

@@ -0,0 +1,35 @@
+from datetime import timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from apps.library.models import TempUpload
+
+
+class Command(BaseCommand):
+    help = "Delete staged uploads older than N hours (default 24)."
+
+    def add_arguments(self, parser):
+        parser.add_argument("--hours", type=int, default=24)
+        parser.add_argument(
+            "--include-completed",
+            action="store_true",
+            help="Also remove completed staging records.",
+        )
+
+    def handle(self, *args, **options):
+        cutoff = timezone.now() - timedelta(hours=options["hours"])
+        queryset = TempUpload.objects.filter(created_at__lt=cutoff)
+        if not options["include_completed"]:
+            queryset = queryset.exclude(status=TempUpload.STATUS_COMPLETED)
+
+        removed = 0
+        for temp in queryset:
+            if temp.file:
+                temp.file.delete(save=False)
+            temp.delete()
+            removed += 1
+
+        self.stdout.write(
+            self.style.SUCCESS(f"Removed {removed} staged upload(s).")
+        )

backend/apps/library/migrations/0004_tempupload.py

@@ -0,0 +1,42 @@
+# Generated by Django 6.1.1 on 2026-09-17 16:06
+
+import django.db.models.deletion
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0003_mediaitem_e621_data_mediaitem_e621_post_id'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TempUpload',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('file', models.FileField(blank=True, upload_to='uploads/temp/')),
+                ('original_filename', models.CharField(max_length=255)),
+                ('md5', models.CharField(db_index=True, max_length=32)),
+                ('size', models.BigIntegerField(default=0)),
+                ('status', models.CharField(choices=[('pending', 'Pending'), ('visual_match', 'Visual Similarity Detected'), ('completed', 'Completed'), ('error', 'Error')], default='pending', max_length=20)),
+                ('resolution', models.CharField(blank=True, default='', max_length=20)),
+                ('e621_post_id', models.IntegerField(blank=True, null=True)),
+                ('e621_data', models.JSONField(blank=True, null=True)),
+                ('custom_rating', models.CharField(blank=True, default='', max_length=1)),
+                ('custom_tags', models.JSONField(blank=True, default=list)),
+                ('custom_notes', models.TextField(blank=True, default='')),
+                ('iqdb_data', models.JSONField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('library_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='temp_uploads', to='library.mediaitem')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='temp_uploads', to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-created_at'],
+            },
+        ),
+    ]

backend/apps/library/models.py

@@ -1,6 +1,8 @@
 from django.conf import settings
 from django.db import models
 
+import uuid
+
 
 class MediaItem(models.Model):
     """A logical media file, identified by its MD5 fingerprint."""
@@ -57,3 +59,59 @@ class MediaLocation(models.Model):
 
     def __str__(self):
         return self.rel_path
+
+
+class TempUpload(models.Model):
+    """A staged upload that has not been accepted into the library yet."""
+
+    STATUS_PENDING = "pending"
+    STATUS_VISUAL_MATCH = "visual_match"
+    STATUS_COMPLETED = "completed"
+    STATUS_ERROR = "error"
+    STATUS_CHOICES = [
+        (STATUS_PENDING, "Pending"),
+        (STATUS_VISUAL_MATCH, "Visual Similarity Detected"),
+        (STATUS_COMPLETED, "Completed"),
+        (STATUS_ERROR, "Error"),
+    ]
+
+    RESOLUTION_AUTO_MD5 = "auto_md5"
+    RESOLUTION_DUPLICATE = "duplicate"
+    RESOLUTION_LINKED = "linked"
+    RESOLUTION_CUSTOM = "custom"
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        related_name="temp_uploads",
+    )
+    file = models.FileField(upload_to="uploads/temp/", blank=True)
+    original_filename = models.CharField(max_length=255)
+    md5 = models.CharField(max_length=32, db_index=True)
+    size = models.BigIntegerField(default=0)
+    status = models.CharField(
+        max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
+    )
+    resolution = models.CharField(max_length=20, blank=True, default="")
+    e621_post_id = models.IntegerField(null=True, blank=True)
+    e621_data = models.JSONField(null=True, blank=True)
+    custom_rating = models.CharField(max_length=1, blank=True, default="")
+    custom_tags = models.JSONField(default=list, blank=True)
+    custom_notes = models.TextField(blank=True, default="")
+    iqdb_data = models.JSONField(null=True, blank=True)
+    library_item = models.ForeignKey(
+        MediaItem,
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="temp_uploads",
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    class Meta:
+        ordering = ["-created_at"]
+
+    def __str__(self):
+        return f"{self.original_filename} ({self.status})"

backend/apps/library/serializers.py

@@ -2,7 +2,7 @@ import os
 
 from rest_framework import serializers
 
-from .models import MediaItem, MediaLocation
+from .models import MediaItem, MediaLocation, TempUpload
 from .services import VIDEO_EXTENSIONS
 
 
@@ -92,3 +92,37 @@ class MediaItemSerializer(serializers.ModelSerializer):
 
     def get_kind(self, obj):
         return "video" if "." + self.get_extension(obj) in VIDEO_EXTENSIONS else "image"
+
+
+class TempUploadSerializer(serializers.ModelSerializer):
+    temp_id = serializers.UUIDField(source="id", read_only=True)
+    library_j_id = serializers.SerializerMethodField()
+    file_url = serializers.SerializerMethodField()
+
+    class Meta:
+        model = TempUpload
+        fields = [
+            "temp_id",
+            "original_filename",
+            "md5",
+            "size",
+            "status",
+            "resolution",
+            "e621_post_id",
+            "e621_data",
+            "custom_rating",
+            "custom_tags",
+            "custom_notes",
+            "iqdb_data",
+            "library_j_id",
+            "file_url",
+            "created_at",
+            "updated_at",
+        ]
+        read_only_fields = fields
+
+    def get_library_j_id(self, obj):
+        return f"J-{obj.library_item_id}" if obj.library_item_id else None
+
+    def get_file_url(self, obj):
+        return f"/api/uploads/{obj.id}/file/" if obj.file else None

backend/apps/library/services.py

@@ -226,6 +226,26 @@ def trim_e621_post(post):
     }
 
 
+def sanitize_iqdb_results(results):
+    """Keep a compact, size-bounded copy of IQDB candidates."""
+    cleaned = []
+    for result in results[:10]:
+        if not isinstance(result, dict):
+            continue
+        score = result.get("score")
+        post_id = result.get("post_id")
+        entry = {
+            "post_id": post_id if isinstance(post_id, int) else None,
+            "score": float(score) if isinstance(score, (int, float)) else None,
+            "preview_url": str(result.get("preview_url") or "")[:500] or None,
+            "rating": str(result.get("rating") or "")[:1] or None,
+            "md5": str(result.get("md5") or "")[:32] or None,
+        }
+        if entry["post_id"] or entry["preview_url"]:
+            cleaned.append(entry)
+    return cleaned
+
+
 def generate_video_thumbnail(md5, path):
     """Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
     if not shutil.which("ffmpeg"):

backend/apps/library/uploads.py

@@ -0,0 +1,212 @@
+"""Staged uploads: files land in a temp folder until they are resolved.
+
+Flow:
+- POST /api/uploads/            stage a file (computes MD5, detects duplicates)
+- GET  /api/uploads/            list my staged uploads
+- GET  /api/uploads/<id>/file/  serve the staged file (previews, IQDB)
+- POST /api/uploads/<id>/iqdb/  store IQDB candidates -> visual_match
+- POST /api/uploads/<id>/resolve/ link to a post or apply custom metadata
+- DELETE /api/uploads/<id>/     discard a staged upload / dismiss a record
+"""
+
+import logging
+import shutil
+from pathlib import Path
+
+from django.conf import settings
+from django.http import Http404
+from rest_framework import mixins, status, viewsets
+from rest_framework.decorators import action
+from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
+from rest_framework.response import Response
+
+from . import services
+from .models import MediaItem, TempUpload
+from .permissions import CanUpload
+from .serializers import TempUploadSerializer
+
+logger = logging.getLogger(__name__)
+
+
+def complete_temp_upload(temp):
+    """Move a staged file into the watched folder and index it."""
+    folder = Path(settings.WATCHED_FOLDER)
+    if not temp.file:
+        raise ValueError("staged file is missing")
+
+    destination = services.unique_destination(
+        folder, temp.original_filename or "upload"
+    )
+    with temp.file.open("rb") as source, open(destination, "wb") as target:
+        shutil.copyfileobj(source, target)
+
+    item, _, _ = services.index_file(destination, folder)
+    temp.file.delete(save=False)
+    temp.library_item = item
+    temp.status = TempUpload.STATUS_COMPLETED
+
+    update_fields = []
+    if item.uploaded_by_id is None:
+        item.uploaded_by = temp.user
+        update_fields.append("uploaded_by")
+    if temp.e621_post_id is not None:
+        item.e621_post_id = temp.e621_post_id
+        item.e621_data = temp.e621_data
+        update_fields += ["e621_post_id", "e621_data"]
+    if temp.custom_rating in {"s", "q", "e"} and not item.rating:
+        item.rating = temp.custom_rating
+        update_fields.append("rating")
+    if temp.custom_tags:
+        item.tags = temp.custom_tags
+        update_fields.append("tags")
+    if temp.custom_notes:
+        item.notes = temp.custom_notes
+        update_fields.append("notes")
+    if update_fields:
+        item.save(update_fields=update_fields + ["updated_at"])
+
+    temp.save()
+    return item
+
+
+class TempUploadViewSet(
+    mixins.ListModelMixin,
+    mixins.RetrieveModelMixin,
+    mixins.DestroyModelMixin,
+    viewsets.GenericViewSet,
+):
+    serializer_class = TempUploadSerializer
+    permission_classes = [CanUpload]
+    parser_classes = [MultiPartParser, FormParser, JSONParser]
+    http_method_names = ["get", "post", "delete", "head", "options"]
+
+    def get_queryset(self):
+        queryset = TempUpload.objects.select_related("library_item")
+        user = self.request.user
+        if not (user.is_staff or user.is_superuser):
+            queryset = queryset.filter(user=user)
+        return queryset
+
+    def create(self, request):
+        upload = request.FILES.get("file")
+        if upload is None:
+            return Response(
+                {"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
+            )
+        extension = Path(upload.name).suffix.lower()
+        if extension not in services.ALLOWED_EXTENSIONS:
+            return Response(
+                {"detail": f"Unsupported file type: {extension or 'unknown'}"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        temp = TempUpload.objects.create(
+            user=request.user,
+            file=upload,
+            original_filename=upload.name,
+            size=upload.size,
+        )
+        temp.md5 = services.compute_md5(temp.file.path)
+
+        existing = MediaItem.objects.filter(md5=temp.md5).first()
+        if existing is not None:
+            temp.status = TempUpload.STATUS_COMPLETED
+            temp.resolution = TempUpload.RESOLUTION_DUPLICATE
+            temp.library_item = existing
+            temp.file.delete(save=False)
+        temp.save()
+        return Response(
+            TempUploadSerializer(temp).data, status=status.HTTP_201_CREATED
+        )
+
+    @action(detail=True, methods=["get"])
+    def file(self, request, pk=None):
+        temp = self.get_object()
+        if not temp.file:
+            raise Http404
+        return services.serve_file(request, temp.file.path)
+
+    @action(detail=True, methods=["post"])
+    def iqdb(self, request, pk=None):
+        temp = self.get_object()
+        results = request.data.get("results")
+        if not isinstance(results, list):
+            return Response(
+                {"detail": "results must be a list."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        temp.iqdb_data = services.sanitize_iqdb_results(results)
+        if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data:
+            temp.status = TempUpload.STATUS_VISUAL_MATCH
+        temp.save(update_fields=["iqdb_data", "status", "updated_at"])
+        return Response(TempUploadSerializer(temp).data)
+
+    @action(detail=True, methods=["post"])
+    def resolve(self, request, pk=None):
+        temp = self.get_object()
+        if temp.status == TempUpload.STATUS_COMPLETED:
+            return Response(
+                {"detail": "This upload is already in the library."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        mode = str(request.data.get("mode") or "").strip()
+        if mode == "link":
+            post = request.data.get("post")
+            post_id = request.data.get("post_id")
+            trimmed = services.trim_e621_post(post)
+            has_numeric_id = post_id is not None and str(post_id).isdigit()
+            if trimmed is None and not has_numeric_id:
+                return Response(
+                    {"detail": "A post payload or post_id is required."},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+            if trimmed is not None and trimmed.get("id") is not None:
+                temp.e621_post_id = int(trimmed["id"])
+            elif has_numeric_id:
+                temp.e621_post_id = int(post_id)
+            temp.e621_data = trimmed
+            temp.resolution = (
+                TempUpload.RESOLUTION_AUTO_MD5
+                if request.data.get("auto")
+                else TempUpload.RESOLUTION_LINKED
+            )
+            rating = str(request.data.get("rating") or "")
+            if rating in {"s", "q", "e"}:
+                temp.custom_rating = rating
+            tags = services.parse_tags(request.data.get("tags"))
+            if tags is not None:
+                temp.custom_tags = tags
+            notes = request.data.get("notes")
+            if notes is not None:
+                temp.custom_notes = str(notes)
+        elif mode == "custom":
+            rating = str(request.data.get("rating") or "")
+            temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
+            temp.custom_tags = services.parse_tags(request.data.get("tags")) or []
+            temp.custom_notes = str(request.data.get("notes") or "")
+            temp.resolution = TempUpload.RESOLUTION_CUSTOM
+        else:
+            return Response(
+                {"detail": "mode must be 'link' or 'custom'."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        temp.save()
+        try:
+            complete_temp_upload(temp)
+        except Exception as exc:  # noqa: BLE001 - report completion failures
+            logger.exception("Could not complete staged upload %s", temp.id)
+            temp.status = TempUpload.STATUS_ERROR
+            temp.save(update_fields=["status", "updated_at"])
+            return Response(
+                {"detail": f"Could not finish the upload: {exc}"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        temp.refresh_from_db()
+        return Response(TempUploadSerializer(temp).data)
+
+    def perform_destroy(self, instance):
+        if instance.file:
+            instance.file.delete(save=False)
+        instance.delete()

backend/apps/library/urls.py

@@ -1,14 +1,15 @@
 from django.urls import include, path
 from rest_framework.routers import DefaultRouter
 
-from .views import DownloadToLibraryView, MediaItemViewSet, UploadView
+from .uploads import TempUploadViewSet
+from .views import DownloadToLibraryView, MediaItemViewSet
 
 router = DefaultRouter()
 router.register("files", MediaItemViewSet, basename="file")
+router.register("uploads", TempUploadViewSet, basename="upload")
 
 urlpatterns = [
     path("", include(router.urls)),
-    path("uploads/", UploadView.as_view(), name="upload"),
     path(
         "online/downloads/",
         DownloadToLibraryView.as_view(),

backend/apps/library/views.py

@@ -8,7 +8,6 @@ from django.http import Http404
 from django.shortcuts import get_object_or_404
 from rest_framework import mixins, status, viewsets
 from rest_framework.decorators import action
-from rest_framework.parsers import FormParser, MultiPartParser
 from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
 from rest_framework.response import Response
 from rest_framework.views import APIView
@@ -165,55 +164,6 @@ class MediaItemViewSet(
         )
 
 
-class UploadView(APIView):
-    permission_classes = [CanUpload]
-    parser_classes = [MultiPartParser, FormParser]
-
-    def post(self, request):
-        upload = request.FILES.get("file")
-        if upload is None:
-            return Response(
-                {"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
-            )
-        folder = Path(settings.WATCHED_FOLDER)
-        destination = services.unique_destination(folder, upload.name)
-        with open(destination, "wb") as output:
-            for chunk in upload.chunks():
-                output.write(chunk)
-        try:
-            item, created_item, _ = services.index_file(destination, folder)
-        except Exception as exc:  # noqa: BLE001 - report indexing failures
-            destination.unlink(missing_ok=True)
-            return Response(
-                {"detail": f"Could not index upload: {exc}"},
-                status=status.HTTP_400_BAD_REQUEST,
-            )
-
-        update_fields = []
-        if created_item and item.uploaded_by_id is None:
-            item.uploaded_by = request.user
-            update_fields.append("uploaded_by")
-        rating = request.data.get("rating") or ""
-        if rating in {"s", "q", "e"} and item.rating != rating:
-            item.rating = rating
-            update_fields.append("rating")
-        tags = services.parse_tags(request.data.get("tags"))
-        if tags is not None:
-            item.tags = tags
-            update_fields.append("tags")
-        notes = request.data.get("notes")
-        if notes is not None:
-            item.notes = str(notes)
-            update_fields.append("notes")
-        if update_fields:
-            item.save(update_fields=update_fields + ["updated_at"])
-
-        return Response(
-            MediaItemSerializer(item, context={"request": request}).data,
-            status=status.HTTP_201_CREATED,
-        )
-
-
 class DownloadToLibraryView(APIView):
     """Fetch an e621 file into the watched folder and index it."""