"""Staged uploads: files land in a temp folder until they are resolved. Flow: - POST /api/uploads/ stage a file (computes MD5, detects duplicates) - GET /api/uploads/ list my staged uploads - GET /api/uploads//file/ serve the staged file (previews, IQDB) - POST /api/uploads//iqdb/ store IQDB candidates -> visual_match - POST /api/uploads//resolve/ link to a post or apply custom metadata - DELETE /api/uploads// discard a staged upload / dismiss a record """ import logging import shutil from pathlib import Path from urllib.parse import urlparse from django.conf import settings from django.contrib.auth import get_user_model from django.core import signing from django.http import Http404 from django.utils import timezone from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.permissions import AllowAny from rest_framework.response import Response from . import services from .models import MediaItem, TempUpload from .permissions import CanUpload from .serializers import TempUploadSerializer from .tools import HASH_FIELDS, hashes_similarity logger = logging.getLogger(__name__) def find_library_matches(path, limit=10, user=None, request=None): """Library items visually similar to a staged file.""" hashes = services.compute_visual_hashes(path) if not hashes: return [] algorithms = list(HASH_FIELDS) threshold = settings.VISUAL_MATCH_THRESHOLD matches = [] for item in MediaItem.objects.prefetch_related("locations"): similarity = hashes_similarity( hashes, {field: getattr(item, field, "") for field in algorithms}, algorithms, threshold, ) if similarity is None: continue location = item.locations.first() matches.append( { "j_id": f"J-{item.id}", "filename": Path(location.rel_path).name if location else item.md5, "similarity": round(similarity * 100, 1), "thumbnail_url": services.signed_media_url( item, user, "thumbnail", request=request ), } ) matches.sort(key=lambda entry: entry["similarity"], reverse=True) return matches[:limit] def complete_temp_upload(temp, download_url=None): """Index the upload into the library. With a ``download_url`` the e621 original is fetched and the staged copy is dropped; otherwise the staged file itself moves into the library. """ folder = Path(settings.WATCHED_FOLDER) if download_url: file_data = {} if isinstance(temp.e621_data, dict): file_data = temp.e621_data.get("file") or {} extension = Path(urlparse(download_url).path).suffix if not extension: extension = Path(temp.original_filename).suffix md5 = file_data.get("md5") name = f"{md5}{extension}" if md5 else (temp.original_filename or "download") destination = services.unique_destination(folder, name) try: services.download_file(download_url, destination) except Exception: destination.unlink(missing_ok=True) raise item, _, location, _ = services.index_file(destination, folder) services.rename_location_to_j_id(item, location) services.ensure_visual_hashes(item) if temp.file: temp.file.delete(save=False) else: if not temp.file: raise ValueError("staged file is missing") destination = services.unique_destination( folder, temp.original_filename or "upload" ) with temp.file.open("rb") as source, open(destination, "wb") as target: shutil.copyfileobj(source, target) item, _, location, _ = services.index_file(destination, folder) services.rename_location_to_j_id(item, location) services.ensure_visual_hashes(item) temp.file.delete(save=False) temp.library_item = item temp.status = TempUpload.STATUS_COMPLETED update_fields = [] if item.uploaded_by_id is None: item.uploaded_by = temp.user update_fields.append("uploaded_by") if temp.e621_post_id is not None: item.e621_post_id = temp.e621_post_id item.e621_data = temp.e621_data item.e621_match_status = MediaItem.E621_MATCHED item.e621_checked_at = timezone.now() update_fields += [ "e621_post_id", "e621_data", "e621_match_status", "e621_checked_at", ] if temp.custom_rating in {"s", "q", "e"} and not item.rating: item.rating = temp.custom_rating update_fields.append("rating") if temp.custom_tags: item.tags = temp.custom_tags update_fields.append("tags") if temp.custom_notes: item.notes = temp.custom_notes update_fields.append("notes") if update_fields: item.save(update_fields=update_fields + ["updated_at"]) temp.save() return item class TempUploadViewSet( mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet, ): serializer_class = TempUploadSerializer permission_classes = [CanUpload] parser_classes = [MultiPartParser, FormParser, JSONParser] # Unpaginated: the board shows every staged upload (69-file batches were # silently cut to the API's 48-item page). pagination_class = None http_method_names = ["get", "post", "delete", "head", "options"] def get_queryset(self): queryset = TempUpload.objects.select_related("library_item") user = self.request.user if not user.is_app_staff: queryset = queryset.filter(user=user) return queryset def create(self, request): upload = request.FILES.get("file") if upload is None: return Response( {"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST ) extension = Path(upload.name).suffix.lower() if extension not in services.ALLOWED_EXTENSIONS: return Response( {"detail": f"Unsupported file type: {extension or 'unknown'}"}, status=status.HTTP_400_BAD_REQUEST, ) temp = TempUpload.objects.create( user=request.user, file=upload, original_filename=upload.name, size=upload.size, ) temp.md5 = services.compute_md5(temp.file.path) existing = MediaItem.objects.filter(md5=temp.md5).first() if existing is not None: temp.status = TempUpload.STATUS_COMPLETED temp.resolution = TempUpload.RESOLUTION_DUPLICATE temp.library_item = existing temp.file.delete(save=False) else: matches = find_library_matches( temp.file.path, user=request.user, request=request ) if matches: temp.visual_matches = matches temp.status = TempUpload.STATUS_VISUAL_MATCH temp.save() return Response( self.get_serializer(temp).data, status=status.HTTP_201_CREATED ) @action(detail=True, methods=["get", "head"], permission_classes=[AllowAny]) def file(self, request, pk=None): """Serve the staged file; accepts a signed URL for media tags.""" user = request.user if request.user.is_authenticated else None if user is None: signature = request.query_params.get("sig") if signature: try: payload = signing.loads( signature, salt=services.UPLOAD_FILE_SALT, max_age=86400, ) except signing.BadSignature: payload = None if payload and str(payload.get("temp")) == str(pk): user = ( get_user_model() .objects.filter(pk=payload.get("user")) .first() ) if user is None or not user.can_upload: return Response( {"detail": "Authentication required."}, status=status.HTTP_401_UNAUTHORIZED, ) temp = TempUpload.objects.filter(pk=pk).first() is_owner = temp is not None and temp.user_id == user.id if temp is None or not (is_owner or user.is_app_staff): raise Http404 if not temp.file: raise Http404 return services.serve_file(request, temp.file.path) @action(detail=True, methods=["post"]) def iqdb(self, request, pk=None): temp = self.get_object() results = request.data.get("results") if not isinstance(results, list): return Response( {"detail": "results must be a list."}, status=status.HTTP_400_BAD_REQUEST, ) temp.iqdb_data = services.sanitize_iqdb_results(results) if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data: temp.status = TempUpload.STATUS_VISUAL_MATCH temp.save(update_fields=["iqdb_data", "status", "updated_at"]) return Response(self.get_serializer(temp).data) @action(detail=True, methods=["post"]) def resolve(self, request, pk=None): temp = self.get_object() if temp.status == TempUpload.STATUS_COMPLETED: return Response( {"detail": "This upload is already in the library."}, status=status.HTTP_400_BAD_REQUEST, ) mode = str(request.data.get("mode") or "").strip() download_url = None if mode == "link": post = request.data.get("post") post_id = request.data.get("post_id") trimmed = services.trim_e621_post(post) has_numeric_id = post_id is not None and str(post_id).isdigit() if trimmed is None and not has_numeric_id: return Response( {"detail": "A post payload or post_id is required."}, status=status.HTTP_400_BAD_REQUEST, ) if trimmed is not None and trimmed.get("id") is not None: temp.e621_post_id = int(trimmed["id"]) elif has_numeric_id: temp.e621_post_id = int(post_id) temp.e621_data = trimmed temp.resolution = ( TempUpload.RESOLUTION_AUTO_MD5 if request.data.get("auto") else TempUpload.RESOLUTION_LINKED ) rating = str(request.data.get("rating") or "") if rating in {"s", "q", "e"}: temp.custom_rating = rating tags = services.parse_tags(request.data.get("tags")) if tags is not None: temp.custom_tags = tags notes = request.data.get("notes") if notes is not None: temp.custom_notes = str(notes) # Prefer the e621 original; fall back to the staged file when the # post has no URL or its file is byte-identical to the upload. file_data = (trimmed or {}).get("file") or {} candidate_url = str( request.data.get("file_url") or file_data.get("url") or "" ).strip() post_md5 = str(file_data.get("md5") or "").strip().lower() if candidate_url and post_md5 and post_md5 == temp.md5.lower(): candidate_url = "" download_url = candidate_url or None elif mode == "custom": rating = str(request.data.get("rating") or "") temp.custom_rating = rating if rating in {"s", "q", "e"} else "" temp.custom_tags = services.parse_tags(request.data.get("tags")) or [] temp.custom_notes = str(request.data.get("notes") or "") temp.resolution = TempUpload.RESOLUTION_CUSTOM else: return Response( {"detail": "mode must be 'link' or 'custom'."}, status=status.HTTP_400_BAD_REQUEST, ) temp.save() try: complete_temp_upload(temp, download_url=download_url) except Exception as exc: # noqa: BLE001 - report completion failures logger.exception("Could not complete staged upload %s", temp.id) temp.status = TempUpload.STATUS_ERROR temp.save(update_fields=["status", "updated_at"]) return Response( {"detail": f"Could not finish the upload: {exc}"}, status=status.HTTP_400_BAD_REQUEST, ) temp.refresh_from_db() return Response(self.get_serializer(temp).data) def perform_destroy(self, instance): if instance.file: instance.file.delete(save=False) instance.delete()