JakeBreath
99f617d296
Footer: - Left is now 'Backend Storage:' with a capacity bar (blue, peach at 80%, red at 95% per DESIGN.md) and a used/total/free tooltip; the watched folder path is no longer printed. /api/status/ returns a compact storage summary instead of the path (the full storage page still shows paths to authenticated users). - Centre shows the backend API origin (empty = same origin). Staff get a link to /setup to point the browser elsewhere; everyone else sees it as plain text. The Account 'Backend connection' card is gone — this is installation plumbing, not a per-user setting. - Design spec updated to match. Staff role: - The custom role did nothing on several endpoints that only accepted Django's is_staff/is_superuser. One canonical check now exists: User.is_app_staff (superuser, Django staff, or the staff role), used by the stats/users APIs, item object permissions, can_delete, upload/ similarity/download/match querysets, and the management commands (which also pick staff-role accounts for e621 sync/match and file ownership). Verified with a role-only staff account (is_staff/is_superuser false): stats/users 200, all 32 downloads + 2 scans visible, others' items editable; the same account as role=user gets 403 for all of those.
332 lines
13 KiB
Python
332 lines
13 KiB
Python
"""Staged uploads: files land in a temp folder until they are resolved.
|
|
|
|
Flow:
|
|
- POST /api/uploads/ stage a file (computes MD5, detects duplicates)
|
|
- GET /api/uploads/ list my staged uploads
|
|
- GET /api/uploads/<id>/file/ serve the staged file (previews, IQDB)
|
|
- POST /api/uploads/<id>/iqdb/ store IQDB candidates -> visual_match
|
|
- POST /api/uploads/<id>/resolve/ link to a post or apply custom metadata
|
|
- DELETE /api/uploads/<id>/ discard a staged upload / dismiss a record
|
|
"""
|
|
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
|
|
from django.conf import settings
|
|
from django.contrib.auth import get_user_model
|
|
from django.core import signing
|
|
from django.http import Http404
|
|
from django.utils import timezone
|
|
from rest_framework import mixins, status, viewsets
|
|
from rest_framework.decorators import action
|
|
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
|
|
from rest_framework.permissions import AllowAny
|
|
from rest_framework.response import Response
|
|
|
|
from . import services
|
|
from .models import MediaItem, TempUpload
|
|
from .permissions import CanUpload
|
|
from .serializers import TempUploadSerializer
|
|
from .tools import HASH_FIELDS, hashes_similarity
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def find_library_matches(path, limit=10, user=None, request=None):
|
|
"""Library items visually similar to a staged file."""
|
|
hashes = services.compute_visual_hashes(path)
|
|
if not hashes:
|
|
return []
|
|
algorithms = list(HASH_FIELDS)
|
|
threshold = settings.VISUAL_MATCH_THRESHOLD
|
|
matches = []
|
|
for item in MediaItem.objects.prefetch_related("locations"):
|
|
similarity = hashes_similarity(
|
|
hashes,
|
|
{field: getattr(item, field, "") for field in algorithms},
|
|
algorithms,
|
|
threshold,
|
|
)
|
|
if similarity is None:
|
|
continue
|
|
location = item.locations.first()
|
|
matches.append(
|
|
{
|
|
"j_id": f"J-{item.id}",
|
|
"filename": Path(location.rel_path).name if location else item.md5,
|
|
"similarity": round(similarity * 100, 1),
|
|
"thumbnail_url": services.signed_media_url(
|
|
item, user, "thumbnail", request=request
|
|
),
|
|
}
|
|
)
|
|
matches.sort(key=lambda entry: entry["similarity"], reverse=True)
|
|
return matches[:limit]
|
|
|
|
|
|
def complete_temp_upload(temp, download_url=None):
|
|
"""Index the upload into the library.
|
|
|
|
With a ``download_url`` the e621 original is fetched and the staged copy
|
|
is dropped; otherwise the staged file itself moves into the library.
|
|
"""
|
|
folder = Path(settings.WATCHED_FOLDER)
|
|
|
|
if download_url:
|
|
file_data = {}
|
|
if isinstance(temp.e621_data, dict):
|
|
file_data = temp.e621_data.get("file") or {}
|
|
extension = Path(urlparse(download_url).path).suffix
|
|
if not extension:
|
|
extension = Path(temp.original_filename).suffix
|
|
md5 = file_data.get("md5")
|
|
name = f"{md5}{extension}" if md5 else (temp.original_filename or "download")
|
|
destination = services.unique_destination(folder, name)
|
|
try:
|
|
services.download_file(download_url, destination)
|
|
except Exception:
|
|
destination.unlink(missing_ok=True)
|
|
raise
|
|
item, _, location, _ = services.index_file(destination, folder)
|
|
services.rename_location_to_j_id(item, location)
|
|
services.ensure_visual_hashes(item)
|
|
if temp.file:
|
|
temp.file.delete(save=False)
|
|
else:
|
|
if not temp.file:
|
|
raise ValueError("staged file is missing")
|
|
destination = services.unique_destination(
|
|
folder, temp.original_filename or "upload"
|
|
)
|
|
with temp.file.open("rb") as source, open(destination, "wb") as target:
|
|
shutil.copyfileobj(source, target)
|
|
item, _, location, _ = services.index_file(destination, folder)
|
|
services.rename_location_to_j_id(item, location)
|
|
services.ensure_visual_hashes(item)
|
|
temp.file.delete(save=False)
|
|
|
|
temp.library_item = item
|
|
temp.status = TempUpload.STATUS_COMPLETED
|
|
|
|
update_fields = []
|
|
if item.uploaded_by_id is None:
|
|
item.uploaded_by = temp.user
|
|
update_fields.append("uploaded_by")
|
|
if temp.e621_post_id is not None:
|
|
item.e621_post_id = temp.e621_post_id
|
|
item.e621_data = temp.e621_data
|
|
item.e621_match_status = MediaItem.E621_MATCHED
|
|
item.e621_checked_at = timezone.now()
|
|
update_fields += [
|
|
"e621_post_id",
|
|
"e621_data",
|
|
"e621_match_status",
|
|
"e621_checked_at",
|
|
]
|
|
if temp.custom_rating in {"s", "q", "e"} and not item.rating:
|
|
item.rating = temp.custom_rating
|
|
update_fields.append("rating")
|
|
if temp.custom_tags:
|
|
item.tags = temp.custom_tags
|
|
update_fields.append("tags")
|
|
if temp.custom_notes:
|
|
item.notes = temp.custom_notes
|
|
update_fields.append("notes")
|
|
if update_fields:
|
|
item.save(update_fields=update_fields + ["updated_at"])
|
|
|
|
temp.save()
|
|
return item
|
|
|
|
|
|
class TempUploadViewSet(
|
|
mixins.ListModelMixin,
|
|
mixins.RetrieveModelMixin,
|
|
mixins.DestroyModelMixin,
|
|
viewsets.GenericViewSet,
|
|
):
|
|
serializer_class = TempUploadSerializer
|
|
permission_classes = [CanUpload]
|
|
parser_classes = [MultiPartParser, FormParser, JSONParser]
|
|
http_method_names = ["get", "post", "delete", "head", "options"]
|
|
|
|
def get_queryset(self):
|
|
queryset = TempUpload.objects.select_related("library_item")
|
|
user = self.request.user
|
|
if not user.is_app_staff:
|
|
queryset = queryset.filter(user=user)
|
|
return queryset
|
|
|
|
def create(self, request):
|
|
upload = request.FILES.get("file")
|
|
if upload is None:
|
|
return Response(
|
|
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
|
|
)
|
|
extension = Path(upload.name).suffix.lower()
|
|
if extension not in services.ALLOWED_EXTENSIONS:
|
|
return Response(
|
|
{"detail": f"Unsupported file type: {extension or 'unknown'}"},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
temp = TempUpload.objects.create(
|
|
user=request.user,
|
|
file=upload,
|
|
original_filename=upload.name,
|
|
size=upload.size,
|
|
)
|
|
temp.md5 = services.compute_md5(temp.file.path)
|
|
|
|
existing = MediaItem.objects.filter(md5=temp.md5).first()
|
|
if existing is not None:
|
|
temp.status = TempUpload.STATUS_COMPLETED
|
|
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
|
|
temp.library_item = existing
|
|
temp.file.delete(save=False)
|
|
else:
|
|
matches = find_library_matches(
|
|
temp.file.path, user=request.user, request=request
|
|
)
|
|
if matches:
|
|
temp.visual_matches = matches
|
|
temp.status = TempUpload.STATUS_VISUAL_MATCH
|
|
temp.save()
|
|
return Response(
|
|
self.get_serializer(temp).data, status=status.HTTP_201_CREATED
|
|
)
|
|
|
|
@action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
|
|
def file(self, request, pk=None):
|
|
"""Serve the staged file; accepts a signed URL for media tags."""
|
|
user = request.user if request.user.is_authenticated else None
|
|
if user is None:
|
|
signature = request.query_params.get("sig")
|
|
if signature:
|
|
try:
|
|
payload = signing.loads(
|
|
signature,
|
|
salt=services.UPLOAD_FILE_SALT,
|
|
max_age=86400,
|
|
)
|
|
except signing.BadSignature:
|
|
payload = None
|
|
if payload and str(payload.get("temp")) == str(pk):
|
|
user = (
|
|
get_user_model()
|
|
.objects.filter(pk=payload.get("user"))
|
|
.first()
|
|
)
|
|
if user is None or not user.can_upload:
|
|
return Response(
|
|
{"detail": "Authentication required."},
|
|
status=status.HTTP_401_UNAUTHORIZED,
|
|
)
|
|
|
|
temp = TempUpload.objects.filter(pk=pk).first()
|
|
is_owner = temp is not None and temp.user_id == user.id
|
|
if temp is None or not (is_owner or user.is_app_staff):
|
|
raise Http404
|
|
if not temp.file:
|
|
raise Http404
|
|
return services.serve_file(request, temp.file.path)
|
|
|
|
@action(detail=True, methods=["post"])
|
|
def iqdb(self, request, pk=None):
|
|
temp = self.get_object()
|
|
results = request.data.get("results")
|
|
if not isinstance(results, list):
|
|
return Response(
|
|
{"detail": "results must be a list."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
temp.iqdb_data = services.sanitize_iqdb_results(results)
|
|
if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data:
|
|
temp.status = TempUpload.STATUS_VISUAL_MATCH
|
|
temp.save(update_fields=["iqdb_data", "status", "updated_at"])
|
|
return Response(self.get_serializer(temp).data)
|
|
|
|
@action(detail=True, methods=["post"])
|
|
def resolve(self, request, pk=None):
|
|
temp = self.get_object()
|
|
if temp.status == TempUpload.STATUS_COMPLETED:
|
|
return Response(
|
|
{"detail": "This upload is already in the library."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
mode = str(request.data.get("mode") or "").strip()
|
|
download_url = None
|
|
if mode == "link":
|
|
post = request.data.get("post")
|
|
post_id = request.data.get("post_id")
|
|
trimmed = services.trim_e621_post(post)
|
|
has_numeric_id = post_id is not None and str(post_id).isdigit()
|
|
if trimmed is None and not has_numeric_id:
|
|
return Response(
|
|
{"detail": "A post payload or post_id is required."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
if trimmed is not None and trimmed.get("id") is not None:
|
|
temp.e621_post_id = int(trimmed["id"])
|
|
elif has_numeric_id:
|
|
temp.e621_post_id = int(post_id)
|
|
temp.e621_data = trimmed
|
|
temp.resolution = (
|
|
TempUpload.RESOLUTION_AUTO_MD5
|
|
if request.data.get("auto")
|
|
else TempUpload.RESOLUTION_LINKED
|
|
)
|
|
rating = str(request.data.get("rating") or "")
|
|
if rating in {"s", "q", "e"}:
|
|
temp.custom_rating = rating
|
|
tags = services.parse_tags(request.data.get("tags"))
|
|
if tags is not None:
|
|
temp.custom_tags = tags
|
|
notes = request.data.get("notes")
|
|
if notes is not None:
|
|
temp.custom_notes = str(notes)
|
|
|
|
# Prefer the e621 original; fall back to the staged file when the
|
|
# post has no URL or its file is byte-identical to the upload.
|
|
file_data = (trimmed or {}).get("file") or {}
|
|
candidate_url = str(
|
|
request.data.get("file_url") or file_data.get("url") or ""
|
|
).strip()
|
|
post_md5 = str(file_data.get("md5") or "").strip().lower()
|
|
if candidate_url and post_md5 and post_md5 == temp.md5.lower():
|
|
candidate_url = ""
|
|
download_url = candidate_url or None
|
|
elif mode == "custom":
|
|
rating = str(request.data.get("rating") or "")
|
|
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
|
|
temp.custom_tags = services.parse_tags(request.data.get("tags")) or []
|
|
temp.custom_notes = str(request.data.get("notes") or "")
|
|
temp.resolution = TempUpload.RESOLUTION_CUSTOM
|
|
else:
|
|
return Response(
|
|
{"detail": "mode must be 'link' or 'custom'."},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
|
|
temp.save()
|
|
try:
|
|
complete_temp_upload(temp, download_url=download_url)
|
|
except Exception as exc: # noqa: BLE001 - report completion failures
|
|
logger.exception("Could not complete staged upload %s", temp.id)
|
|
temp.status = TempUpload.STATUS_ERROR
|
|
temp.save(update_fields=["status", "updated_at"])
|
|
return Response(
|
|
{"detail": f"Could not finish the upload: {exc}"},
|
|
status=status.HTTP_400_BAD_REQUEST,
|
|
)
|
|
temp.refresh_from_db()
|
|
return Response(self.get_serializer(temp).data)
|
|
|
|
def perform_destroy(self, instance):
|
|
if instance.file:
|
|
instance.file.delete(save=False)
|
|
instance.delete()
|