Files
J621/backend/apps/library/uploads.py
T
JakeBreath d0e2901c92 Upload pipeline: staging, MD5 auto-match, IQDB, three-column board
Backend:
- TempUpload model: staged files (pending / visual_match / completed /
  error) with resolution, e621 payload, custom metadata and IQDB data
- Files land in a temp folder and only move into the watched library
  folder once resolved; duplicates resolve immediately without a copy
- Endpoints: stage (multipart), list, retrieve, temp file, IQDB save,
  resolve (link to post or custom metadata), discard/dismiss
- cleanup_temp_uploads command for old staged files
- Replaces the old direct-to-library upload endpoint

Frontend:
- Upload page is now a three-column board (Pending & Unmatched /
  Visual Similarity Detected / Auto-uploaded & Indexed)
- After upload: MD5s are batch-checked against e621 and matches
  auto-complete with full post metadata; remaining files run through
  IQDB and move to the similarity column when candidates exist
- Metadata modal with IQDB candidates, post-ID linking and custom
  tags/rating/notes; discard and dismiss actions
- e621 client gains fetchPostsByMd5 and iqdbSearch helpers

Roadmap updated with the completed upload items.
2026-09-17 11:12:03 -05:00

213 lines
7.9 KiB
Python

"""Staged uploads: files land in a temp folder until they are resolved.
Flow:
- POST /api/uploads/ stage a file (computes MD5, detects duplicates)
- GET /api/uploads/ list my staged uploads
- GET /api/uploads/<id>/file/ serve the staged file (previews, IQDB)
- POST /api/uploads/<id>/iqdb/ store IQDB candidates -> visual_match
- POST /api/uploads/<id>/resolve/ link to a post or apply custom metadata
- DELETE /api/uploads/<id>/ discard a staged upload / dismiss a record
"""
import logging
import shutil
from pathlib import Path
from django.conf import settings
from django.http import Http404
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import action
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
from rest_framework.response import Response
from . import services
from .models import MediaItem, TempUpload
from .permissions import CanUpload
from .serializers import TempUploadSerializer
logger = logging.getLogger(__name__)
def complete_temp_upload(temp):
"""Move a staged file into the watched folder and index it."""
folder = Path(settings.WATCHED_FOLDER)
if not temp.file:
raise ValueError("staged file is missing")
destination = services.unique_destination(
folder, temp.original_filename or "upload"
)
with temp.file.open("rb") as source, open(destination, "wb") as target:
shutil.copyfileobj(source, target)
item, _, _ = services.index_file(destination, folder)
temp.file.delete(save=False)
temp.library_item = item
temp.status = TempUpload.STATUS_COMPLETED
update_fields = []
if item.uploaded_by_id is None:
item.uploaded_by = temp.user
update_fields.append("uploaded_by")
if temp.e621_post_id is not None:
item.e621_post_id = temp.e621_post_id
item.e621_data = temp.e621_data
update_fields += ["e621_post_id", "e621_data"]
if temp.custom_rating in {"s", "q", "e"} and not item.rating:
item.rating = temp.custom_rating
update_fields.append("rating")
if temp.custom_tags:
item.tags = temp.custom_tags
update_fields.append("tags")
if temp.custom_notes:
item.notes = temp.custom_notes
update_fields.append("notes")
if update_fields:
item.save(update_fields=update_fields + ["updated_at"])
temp.save()
return item
class TempUploadViewSet(
mixins.ListModelMixin,
mixins.RetrieveModelMixin,
mixins.DestroyModelMixin,
viewsets.GenericViewSet,
):
serializer_class = TempUploadSerializer
permission_classes = [CanUpload]
parser_classes = [MultiPartParser, FormParser, JSONParser]
http_method_names = ["get", "post", "delete", "head", "options"]
def get_queryset(self):
queryset = TempUpload.objects.select_related("library_item")
user = self.request.user
if not (user.is_staff or user.is_superuser):
queryset = queryset.filter(user=user)
return queryset
def create(self, request):
upload = request.FILES.get("file")
if upload is None:
return Response(
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
)
extension = Path(upload.name).suffix.lower()
if extension not in services.ALLOWED_EXTENSIONS:
return Response(
{"detail": f"Unsupported file type: {extension or 'unknown'}"},
status=status.HTTP_400_BAD_REQUEST,
)
temp = TempUpload.objects.create(
user=request.user,
file=upload,
original_filename=upload.name,
size=upload.size,
)
temp.md5 = services.compute_md5(temp.file.path)
existing = MediaItem.objects.filter(md5=temp.md5).first()
if existing is not None:
temp.status = TempUpload.STATUS_COMPLETED
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
temp.library_item = existing
temp.file.delete(save=False)
temp.save()
return Response(
TempUploadSerializer(temp).data, status=status.HTTP_201_CREATED
)
@action(detail=True, methods=["get"])
def file(self, request, pk=None):
temp = self.get_object()
if not temp.file:
raise Http404
return services.serve_file(request, temp.file.path)
@action(detail=True, methods=["post"])
def iqdb(self, request, pk=None):
temp = self.get_object()
results = request.data.get("results")
if not isinstance(results, list):
return Response(
{"detail": "results must be a list."},
status=status.HTTP_400_BAD_REQUEST,
)
temp.iqdb_data = services.sanitize_iqdb_results(results)
if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data:
temp.status = TempUpload.STATUS_VISUAL_MATCH
temp.save(update_fields=["iqdb_data", "status", "updated_at"])
return Response(TempUploadSerializer(temp).data)
@action(detail=True, methods=["post"])
def resolve(self, request, pk=None):
temp = self.get_object()
if temp.status == TempUpload.STATUS_COMPLETED:
return Response(
{"detail": "This upload is already in the library."},
status=status.HTTP_400_BAD_REQUEST,
)
mode = str(request.data.get("mode") or "").strip()
if mode == "link":
post = request.data.get("post")
post_id = request.data.get("post_id")
trimmed = services.trim_e621_post(post)
has_numeric_id = post_id is not None and str(post_id).isdigit()
if trimmed is None and not has_numeric_id:
return Response(
{"detail": "A post payload or post_id is required."},
status=status.HTTP_400_BAD_REQUEST,
)
if trimmed is not None and trimmed.get("id") is not None:
temp.e621_post_id = int(trimmed["id"])
elif has_numeric_id:
temp.e621_post_id = int(post_id)
temp.e621_data = trimmed
temp.resolution = (
TempUpload.RESOLUTION_AUTO_MD5
if request.data.get("auto")
else TempUpload.RESOLUTION_LINKED
)
rating = str(request.data.get("rating") or "")
if rating in {"s", "q", "e"}:
temp.custom_rating = rating
tags = services.parse_tags(request.data.get("tags"))
if tags is not None:
temp.custom_tags = tags
notes = request.data.get("notes")
if notes is not None:
temp.custom_notes = str(notes)
elif mode == "custom":
rating = str(request.data.get("rating") or "")
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
temp.custom_tags = services.parse_tags(request.data.get("tags")) or []
temp.custom_notes = str(request.data.get("notes") or "")
temp.resolution = TempUpload.RESOLUTION_CUSTOM
else:
return Response(
{"detail": "mode must be 'link' or 'custom'."},
status=status.HTTP_400_BAD_REQUEST,
)
temp.save()
try:
complete_temp_upload(temp)
except Exception as exc: # noqa: BLE001 - report completion failures
logger.exception("Could not complete staged upload %s", temp.id)
temp.status = TempUpload.STATUS_ERROR
temp.save(update_fields=["status", "updated_at"])
return Response(
{"detail": f"Could not finish the upload: {exc}"},
status=status.HTTP_400_BAD_REQUEST,
)
temp.refresh_from_db()
return Response(TempUploadSerializer(temp).data)
def perform_destroy(self, instance):
if instance.file:
instance.file.delete(save=False)
instance.delete()