JakeBreath
db74f7ab18
Items flagged hidden_from_guests (blacklisted tags) returned 404 for <img> requests since tags cannot send the auth header. The API now exposes signed raw_url/thumbnail_url fields (mirroring upload previews and avatars), and the SPA uses them in the gallery, detail view, duplicates and delete screens, and upload visual matches.
322 lines
12 KiB
Python
322 lines
12 KiB
Python
"""Staged uploads: files land in a temp folder until they are resolved.
|
|
|
|
Flow:
|
|
- POST /api/uploads/ stage a file (computes MD5, detects duplicates)
|
|
- GET /api/uploads/ list my staged uploads
|
|
- GET /api/uploads/<id>/file/ serve the staged file (previews, IQDB)
|
|
- POST /api/uploads/<id>/iqdb/ store IQDB candidates -> visual_match
|
|
- POST /api/uploads/<id>/resolve/ link to a post or apply custom metadata
|
|
- DELETE /api/uploads/<id>/ discard a staged upload / dismiss a record
|
|
"""
|
|
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
from django.conf import settings
|
|
from django.contrib.auth import get_user_model
|
|
from django.core import signing
|
|
from django.http import Http404
|
|
from rest_framework import mixins, status, viewsets
|
|
from rest_framework.decorators import action
|
|
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
|
|
from rest_framework.permissions import AllowAny
|
|
from rest_framework.response import Response
|
|
|
|
from . import services
|
|
from .models import MediaItem, TempUpload
|
|
from .permissions import CanUpload
|
|
from .serializers import TempUploadSerializer
|
|
from .tools import HASH_FIELDS, hashes_similarity
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def find_library_matches(path, limit=10, user=None):
|
|
"""Library items visually similar to a staged file."""
|
|
hashes = services.compute_visual_hashes(path)
|
|
if not hashes:
|
|
return []
|
|
algorithms = list(HASH_FIELDS)
|
|
threshold = settings.VISUAL_MATCH_THRESHOLD
|
|
matches = []
|
|
for item in MediaItem.objects.prefetch_related("locations"):
|
|
similarity = hashes_similarity(
|
|
hashes,
|
|
{field: getattr(item, field, "") for field in algorithms},
|
|
algorithms,
|
|
threshold,
|
|
)
|
|
if similarity is None:
|
|
continue
|
|
location = item.locations.first()
|
|
matches.append(
|
|
{
|
|
"j_id": f"J-{item.id}",
|
|
"filename": Path(location.rel_path).name if location else item.md5,
|
|
"similarity": round(similarity * 100, 1),
|
|
"thumbnail_url": services.signed_media_url(item, user, "thumbnail"),
|
|
}
|
|
)
|
|
matches.sort(key=lambda entry: entry["similarity"], reverse=True)
|
|
return matches[:limit]
|
|
|
|
|
|
def complete_temp_upload(temp, download_url=None):
|
|
"""Index the upload into the library.
|
|
|
|
With a ``download_url`` the e621 original is fetched and the staged copy
|
|
is dropped; otherwise the staged file itself moves into the library.
|
|
"""
|
|
folder = Path(settings.WATCHED_FOLDER)
|
|
|
|
if download_url:
|
|
file_data = {}
|
|
if isinstance(temp.e621_data, dict):
|
|
file_data = temp.e621_data.get("file") or {}
|
|
extension = Path(urlparse(download_url).path).suffix
|
|
if not extension:
|
|
extension = Path(temp.original_filename).suffix
|
|
md5 = file_data.get("md5")
|
|
name = f"{md5}{extension}" if md5 else (temp.original_filename or "download")
|
|
destination = services.unique_destination(folder, name)
|
|
try:
|
|
services.download_file(download_url, destination)
|
|
except Exception:
|
|
destination.unlink(missing_ok=True)
|
|
raise
|
|
item, _, location, _ = services.index_file(destination, folder)
|
|
services.rename_location_to_j_id(item, location)
|
|
services.ensure_visual_hashes(item)
|
|
if temp.file:
|
|
temp.file.delete(save=False)
|
|
else:
|
|
if not temp.file:
|
|
raise ValueError("staged file is missing")
|
|
destination = services.unique_destination(
|
|
folder, temp.original_filename or "upload"
|
|
)
|
|
with temp.file.open("rb") as source, open(destination, "wb") as target:
|
|
shutil.copyfileobj(source, target)
|
|
item, _, location, _ = services.index_file(destination, folder)
|
|
services.rename_location_to_j_id(item, location)
|
|
services.ensure_visual_hashes(item)
|
|
temp.file.delete(save=False)
|
|
|
|
temp.library_item = item
|
|
temp.status = TempUpload.STATUS_COMPLETED
|
|
|
|
update_fields = []
|
|
if item.uploaded_by_id is None:
|
|
item.uploaded_by = temp.user
|
|
update_fields.append("uploaded_by")
|
|
if temp.e621_post_id is not None:
|
|
item.e621_post_id = temp.e621_post_id
|
|
item.e621_data = temp.e621_data
|
|
update_fields += ["e621_post_id", "e621_data"]
|
|
if temp.custom_rating in {"s", "q", "e"} and not item.rating:
|
|
item.rating = temp.custom_rating
|
|
update_fields.append("rating")
|
|
if temp.custom_tags:
|
|
item.tags = temp.custom_tags
|
|
update_fields.append("tags")
|
|
if temp.custom_notes:
|
|
item.notes = temp.custom_notes
|
|
update_fields.append("notes")
|
|
if update_fields:
|
|
item.save(update_fields=update_fields + ["updated_at"])
|
|
|
|
temp.save()
|
|
return item
|
|
|
|
|
|
class TempUploadViewSet(
|
|
mixins.ListModelMixin,
|
|
mixins.RetrieveModelMixin,
|
|
mixins.DestroyModelMixin,
|
|
viewsets.GenericViewSet,
|
|
):
|
|
serializer_class = TempUploadSerializer
|
|
permission_classes = [CanUpload]
|
|
parser_classes = [MultiPartParser, FormParser, JSONParser]
|
|
http_method_names = ["get", "post", "delete", "head", "options"]
|
|
|
|
def get_queryset(self):
|
|
queryset = TempUpload.objects.select_related("library_item")
|
|
user = self.request.user
|
|
if not (user.is_staff or user.is_superuser):
|
|
queryset = queryset.filter(user=user)
|
|
return queryset
|
|
|
|
def create(self, request):
|
|
upload = request.FILES.get("file")
|
|
if upload is None:
|
|
return Response(
|
|
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
|
|
)
|
|
extension = Path(upload.name).suffix.lower()
|
|
if extension not in services.ALLOWED_EXTENSIONS:
|
|
return Response(
|
|
{"detail": f"Unsupported file type: {extension or 'unknown'}"},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
temp = TempUpload.objects.create(
|
|
user=request.user,
|
|
file=upload,
|
|
original_filename=upload.name,
|
|
size=upload.size,
|
|
)
|
|
temp.md5 = services.compute_md5(temp.file.path)
|
|
|
|
existing = MediaItem.objects.filter(md5=temp.md5).first()
|
|
if existing is not None:
|
|
temp.status = TempUpload.STATUS_COMPLETED
|
|
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
|
|
temp.library_item = existing
|
|
temp.file.delete(save=False)
|
|
else:
|
|
matches = find_library_matches(temp.file.path, user=request.user)
|
|
if matches:
|
|
temp.visual_matches = matches
|
|
temp.status = TempUpload.STATUS_VISUAL_MATCH
|
|
temp.save()
|
|
return Response(
|
|
self.get_serializer(temp).data, status=status.HTTP_201_CREATED
|
|
)
|
|
|
|
@action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
|
|
def file(self, request, pk=None):
|
|
"""Serve the staged file; accepts a signed URL for media tags."""
|
|
user = request.user if request.user.is_authenticated else None
|
|
if user is None:
|
|
signature = request.query_params.get("sig")
|
|
if signature:
|
|
try:
|
|
payload = signing.loads(
|
|
signature,
|
|
salt=services.UPLOAD_FILE_SALT,
|
|
max_age=86400,
|
|
)
|
|
except signing.BadSignature:
|
|
payload = None
|
|
if payload and str(payload.get("temp")) == str(pk):
|
|
user = (
|
|
get_user_model()
|
|
.objects.filter(pk=payload.get("user"))
|
|
.first()
|
|
)
|
|
if user is None or not user.can_upload:
|
|
return Response(
|
|
{"detail": "Authentication required."},
|
|
status=status.HTTP_401_UNAUTHORIZED,
|
|
)
|
|
|
|
temp = TempUpload.objects.filter(pk=pk).first()
|
|
is_owner = temp is not None and temp.user_id == user.id
|
|
if temp is None or not (
|
|
is_owner or user.is_staff or user.is_superuser
|
|
):
|
|
raise Http404
|
|
if not temp.file:
|
|
raise Http404
|
|
return services.serve_file(request, temp.file.path)
|
|
|
|
@action(detail=True, methods=["post"])
|
|
def iqdb(self, request, pk=None):
|
|
temp = self.get_object()
|
|
results = request.data.get("results")
|
|
if not isinstance(results, list):
|
|
return Response(
|
|
{"detail": "results must be a list."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
temp.iqdb_data = services.sanitize_iqdb_results(results)
|
|
if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data:
|
|
temp.status = TempUpload.STATUS_VISUAL_MATCH
|
|
temp.save(update_fields=["iqdb_data", "status", "updated_at"])
|
|
return Response(self.get_serializer(temp).data)
|
|
|
|
@action(detail=True, methods=["post"])
|
|
def resolve(self, request, pk=None):
|
|
temp = self.get_object()
|
|
if temp.status == TempUpload.STATUS_COMPLETED:
|
|
return Response(
|
|
{"detail": "This upload is already in the library."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
mode = str(request.data.get("mode") or "").strip()
|
|
download_url = None
|
|
if mode == "link":
|
|
post = request.data.get("post")
|
|
post_id = request.data.get("post_id")
|
|
trimmed = services.trim_e621_post(post)
|
|
has_numeric_id = post_id is not None and str(post_id).isdigit()
|
|
if trimmed is None and not has_numeric_id:
|
|
return Response(
|
|
{"detail": "A post payload or post_id is required."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
if trimmed is not None and trimmed.get("id") is not None:
|
|
temp.e621_post_id = int(trimmed["id"])
|
|
elif has_numeric_id:
|
|
temp.e621_post_id = int(post_id)
|
|
temp.e621_data = trimmed
|
|
temp.resolution = (
|
|
TempUpload.RESOLUTION_AUTO_MD5
|
|
if request.data.get("auto")
|
|
else TempUpload.RESOLUTION_LINKED
|
|
)
|
|
rating = str(request.data.get("rating") or "")
|
|
if rating in {"s", "q", "e"}:
|
|
temp.custom_rating = rating
|
|
tags = services.parse_tags(request.data.get("tags"))
|
|
if tags is not None:
|
|
temp.custom_tags = tags
|
|
notes = request.data.get("notes")
|
|
if notes is not None:
|
|
temp.custom_notes = str(notes)
|
|
|
|
# Prefer the e621 original; fall back to the staged file when the
|
|
# post has no URL or its file is byte-identical to the upload.
|
|
file_data = (trimmed or {}).get("file") or {}
|
|
candidate_url = str(
|
|
request.data.get("file_url") or file_data.get("url") or ""
|
|
).strip()
|
|
post_md5 = str(file_data.get("md5") or "").strip().lower()
|
|
if candidate_url and post_md5 and post_md5 == temp.md5.lower():
|
|
candidate_url = ""
|
|
download_url = candidate_url or None
|
|
elif mode == "custom":
|
|
rating = str(request.data.get("rating") or "")
|
|
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
|
|
temp.custom_tags = services.parse_tags(request.data.get("tags")) or []
|
|
temp.custom_notes = str(request.data.get("notes") or "")
|
|
temp.resolution = TempUpload.RESOLUTION_CUSTOM
|
|
else:
|
|
return Response(
|
|
{"detail": "mode must be 'link' or 'custom'."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
temp.save()
|
|
try:
|
|
complete_temp_upload(temp, download_url=download_url)
|
|
except Exception as exc: # noqa: BLE001 - report completion failures
|
|
logger.exception("Could not complete staged upload %s", temp.id)
|
|
temp.status = TempUpload.STATUS_ERROR
|
|
temp.save(update_fields=["status", "updated_at"])
|
|
return Response(
|
|
{"detail": f"Could not finish the upload: {exc}"},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
temp.refresh_from_db()
|
|
return Response(self.get_serializer(temp).data)
|
|
|
|
def perform_destroy(self, instance):
|
|
if instance.file:
|
|
instance.file.delete(save=False)
|
|
instance.delete()
|