Upload pipeline: staging, MD5 auto-match, IQDB, three-column board
Backend: - TempUpload model: staged files (pending / visual_match / completed / error) with resolution, e621 payload, custom metadata and IQDB data - Files land in a temp folder and only move into the watched library folder once resolved; duplicates resolve immediately without a copy - Endpoints: stage (multipart), list, retrieve, temp file, IQDB save, resolve (link to post or custom metadata), discard/dismiss - cleanup_temp_uploads command for old staged files - Replaces the old direct-to-library upload endpoint Frontend: - Upload page is now a three-column board (Pending & Unmatched / Visual Similarity Detected / Auto-uploaded & Indexed) - After upload: MD5s are batch-checked against e621 and matches auto-complete with full post metadata; remaining files run through IQDB and move to the similarity column when candidates exist - Metadata modal with IQDB candidates, post-ID linking and custom tags/rating/notes; discard and dismiss actions - e621 client gains fetchPostsByMd5 and iqdbSearch helpers Roadmap updated with the completed upload items.
This commit is contained in:
+22
-23
@@ -37,32 +37,31 @@ they land.
|
||||
- [ ] "Followed" nav entry
|
||||
- [ ] `F` shortcut to favorite a post, tag finder in the Ctrl+K palette
|
||||
|
||||
## 3. Uploads & ingestion pipeline (major rework)
|
||||
## 3. Uploads & ingestion pipeline
|
||||
|
||||
Current upload writes straight into the watched folder. The target flow matches
|
||||
the original app:
|
||||
Files now stage first and are resolved before entering the library.
|
||||
|
||||
- [ ] **Backend staging storage**
|
||||
- [ ] `TempUpload` model: temp_id, user, md5, filename, temp path, status
|
||||
(`pending` / `visual_match` / `completed` / `error`), timestamps
|
||||
- [ ] Files land in a temp folder first; only "completed" moves them into the
|
||||
watched library folder
|
||||
- [ ] Pending uploads list endpoint, delete/discard endpoint
|
||||
- [ ] **Auto-upload / auto-match**
|
||||
- [ ] Compute MD5 on upload; if it already exists in the library, mark as
|
||||
duplicate and resolve automatically (no new file)
|
||||
- [ ] If the MD5 matches an e621 post, auto-complete: fetch and store the post
|
||||
metadata, seed rating/tags, mark as indexed
|
||||
- [ ] Batch MD5 checks against e621 for staged uploads
|
||||
- [ ] **IQDB / visual similarity on upload**
|
||||
- [ ] Run IQDB on staged uploads and surface candidate posts
|
||||
- [ ] Perceptual-hash check against the library → "Visual Similarity Detected"
|
||||
state with side-by-side comparison
|
||||
- [ ] **Upload UI**
|
||||
- [ ] Three-column board: Pending & Unmatched / Visual Similarity Detected /
|
||||
- [x] **Backend staging storage**
|
||||
- [x] `TempUpload` model: user, md5, filename, temp path, status
|
||||
(`pending` / `visual_match` / `completed` / `error`), resolution
|
||||
- [x] Files land in a temp folder first; only completed uploads move into
|
||||
the watched library folder
|
||||
- [x] Pending/upload list endpoint, file serving, discard endpoint
|
||||
- [x] `cleanup_temp_uploads` command for old staged files
|
||||
- [x] **Auto-upload / auto-match**
|
||||
- [x] MD5 computed on staging; exact duplicates resolve immediately
|
||||
- [x] MD5 batch-checked against e621; matches auto-complete with post
|
||||
metadata stored and rating seeded
|
||||
- [x] **IQDB similarity on upload** (SPA-driven)
|
||||
- [x] Automatic + manual IQDB checks with candidate posts
|
||||
- [x] "Visual Similarity Detected" state with candidate picker
|
||||
- [ ] Perceptual-hash comparison against the library (shared with the
|
||||
duplicates engine, still pending)
|
||||
- [x] **Upload UI**
|
||||
- [x] Three-column board: Pending & Unmatched / Visual Similarity Detected /
|
||||
Auto-uploaded & Indexed
|
||||
- [ ] Metadata modal (link to e621 post via search + custom tags/rating/notes)
|
||||
- [ ] Per-file progress plus batch summary
|
||||
- [x] Metadata modal (link to e621 post, IQDB candidates, custom metadata)
|
||||
- [x] Per-file progress plus batch processing indicator
|
||||
|
||||
## 4. Staff tools
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.library.models import TempUpload
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Delete staged uploads older than N hours (default 24)."
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("--hours", type=int, default=24)
|
||||
parser.add_argument(
|
||||
"--include-completed",
|
||||
action="store_true",
|
||||
help="Also remove completed staging records.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
cutoff = timezone.now() - timedelta(hours=options["hours"])
|
||||
queryset = TempUpload.objects.filter(created_at__lt=cutoff)
|
||||
if not options["include_completed"]:
|
||||
queryset = queryset.exclude(status=TempUpload.STATUS_COMPLETED)
|
||||
|
||||
removed = 0
|
||||
for temp in queryset:
|
||||
if temp.file:
|
||||
temp.file.delete(save=False)
|
||||
temp.delete()
|
||||
removed += 1
|
||||
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f"Removed {removed} staged upload(s).")
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
# Generated by Django 6.1.1 on 2026-09-17 16:06
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('library', '0003_mediaitem_e621_data_mediaitem_e621_post_id'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='TempUpload',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('file', models.FileField(blank=True, upload_to='uploads/temp/')),
|
||||
('original_filename', models.CharField(max_length=255)),
|
||||
('md5', models.CharField(db_index=True, max_length=32)),
|
||||
('size', models.BigIntegerField(default=0)),
|
||||
('status', models.CharField(choices=[('pending', 'Pending'), ('visual_match', 'Visual Similarity Detected'), ('completed', 'Completed'), ('error', 'Error')], default='pending', max_length=20)),
|
||||
('resolution', models.CharField(blank=True, default='', max_length=20)),
|
||||
('e621_post_id', models.IntegerField(blank=True, null=True)),
|
||||
('e621_data', models.JSONField(blank=True, null=True)),
|
||||
('custom_rating', models.CharField(blank=True, default='', max_length=1)),
|
||||
('custom_tags', models.JSONField(blank=True, default=list)),
|
||||
('custom_notes', models.TextField(blank=True, default='')),
|
||||
('iqdb_data', models.JSONField(blank=True, null=True)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('library_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='temp_uploads', to='library.mediaitem')),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='temp_uploads', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -1,6 +1,8 @@
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
import uuid
|
||||
|
||||
|
||||
class MediaItem(models.Model):
|
||||
"""A logical media file, identified by its MD5 fingerprint."""
|
||||
@@ -57,3 +59,59 @@ class MediaLocation(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return self.rel_path
|
||||
|
||||
|
||||
class TempUpload(models.Model):
|
||||
"""A staged upload that has not been accepted into the library yet."""
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_VISUAL_MATCH = "visual_match"
|
||||
STATUS_COMPLETED = "completed"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CHOICES = [
|
||||
(STATUS_PENDING, "Pending"),
|
||||
(STATUS_VISUAL_MATCH, "Visual Similarity Detected"),
|
||||
(STATUS_COMPLETED, "Completed"),
|
||||
(STATUS_ERROR, "Error"),
|
||||
]
|
||||
|
||||
RESOLUTION_AUTO_MD5 = "auto_md5"
|
||||
RESOLUTION_DUPLICATE = "duplicate"
|
||||
RESOLUTION_LINKED = "linked"
|
||||
RESOLUTION_CUSTOM = "custom"
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="temp_uploads",
|
||||
)
|
||||
file = models.FileField(upload_to="uploads/temp/", blank=True)
|
||||
original_filename = models.CharField(max_length=255)
|
||||
md5 = models.CharField(max_length=32, db_index=True)
|
||||
size = models.BigIntegerField(default=0)
|
||||
status = models.CharField(
|
||||
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||
)
|
||||
resolution = models.CharField(max_length=20, blank=True, default="")
|
||||
e621_post_id = models.IntegerField(null=True, blank=True)
|
||||
e621_data = models.JSONField(null=True, blank=True)
|
||||
custom_rating = models.CharField(max_length=1, blank=True, default="")
|
||||
custom_tags = models.JSONField(default=list, blank=True)
|
||||
custom_notes = models.TextField(blank=True, default="")
|
||||
iqdb_data = models.JSONField(null=True, blank=True)
|
||||
library_item = models.ForeignKey(
|
||||
MediaItem,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="temp_uploads",
|
||||
)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.original_filename} ({self.status})"
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from .models import MediaItem, MediaLocation
|
||||
from .models import MediaItem, MediaLocation, TempUpload
|
||||
from .services import VIDEO_EXTENSIONS
|
||||
|
||||
|
||||
@@ -92,3 +92,37 @@ class MediaItemSerializer(serializers.ModelSerializer):
|
||||
|
||||
def get_kind(self, obj):
|
||||
return "video" if "." + self.get_extension(obj) in VIDEO_EXTENSIONS else "image"
|
||||
|
||||
|
||||
class TempUploadSerializer(serializers.ModelSerializer):
|
||||
temp_id = serializers.UUIDField(source="id", read_only=True)
|
||||
library_j_id = serializers.SerializerMethodField()
|
||||
file_url = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = TempUpload
|
||||
fields = [
|
||||
"temp_id",
|
||||
"original_filename",
|
||||
"md5",
|
||||
"size",
|
||||
"status",
|
||||
"resolution",
|
||||
"e621_post_id",
|
||||
"e621_data",
|
||||
"custom_rating",
|
||||
"custom_tags",
|
||||
"custom_notes",
|
||||
"iqdb_data",
|
||||
"library_j_id",
|
||||
"file_url",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
def get_library_j_id(self, obj):
|
||||
return f"J-{obj.library_item_id}" if obj.library_item_id else None
|
||||
|
||||
def get_file_url(self, obj):
|
||||
return f"/api/uploads/{obj.id}/file/" if obj.file else None
|
||||
|
||||
@@ -226,6 +226,26 @@ def trim_e621_post(post):
|
||||
}
|
||||
|
||||
|
||||
def sanitize_iqdb_results(results):
|
||||
"""Keep a compact, size-bounded copy of IQDB candidates."""
|
||||
cleaned = []
|
||||
for result in results[:10]:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
score = result.get("score")
|
||||
post_id = result.get("post_id")
|
||||
entry = {
|
||||
"post_id": post_id if isinstance(post_id, int) else None,
|
||||
"score": float(score) if isinstance(score, (int, float)) else None,
|
||||
"preview_url": str(result.get("preview_url") or "")[:500] or None,
|
||||
"rating": str(result.get("rating") or "")[:1] or None,
|
||||
"md5": str(result.get("md5") or "")[:32] or None,
|
||||
}
|
||||
if entry["post_id"] or entry["preview_url"]:
|
||||
cleaned.append(entry)
|
||||
return cleaned
|
||||
|
||||
|
||||
def generate_video_thumbnail(md5, path):
|
||||
"""Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
|
||||
if not shutil.which("ffmpeg"):
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Staged uploads: files land in a temp folder until they are resolved.
|
||||
|
||||
Flow:
|
||||
- POST /api/uploads/ stage a file (computes MD5, detects duplicates)
|
||||
- GET /api/uploads/ list my staged uploads
|
||||
- GET /api/uploads/<id>/file/ serve the staged file (previews, IQDB)
|
||||
- POST /api/uploads/<id>/iqdb/ store IQDB candidates -> visual_match
|
||||
- POST /api/uploads/<id>/resolve/ link to a post or apply custom metadata
|
||||
- DELETE /api/uploads/<id>/ discard a staged upload / dismiss a record
|
||||
"""
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.http import Http404
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
|
||||
from rest_framework.response import Response
|
||||
|
||||
from . import services
|
||||
from .models import MediaItem, TempUpload
|
||||
from .permissions import CanUpload
|
||||
from .serializers import TempUploadSerializer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def complete_temp_upload(temp):
|
||||
"""Move a staged file into the watched folder and index it."""
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
if not temp.file:
|
||||
raise ValueError("staged file is missing")
|
||||
|
||||
destination = services.unique_destination(
|
||||
folder, temp.original_filename or "upload"
|
||||
)
|
||||
with temp.file.open("rb") as source, open(destination, "wb") as target:
|
||||
shutil.copyfileobj(source, target)
|
||||
|
||||
item, _, _ = services.index_file(destination, folder)
|
||||
temp.file.delete(save=False)
|
||||
temp.library_item = item
|
||||
temp.status = TempUpload.STATUS_COMPLETED
|
||||
|
||||
update_fields = []
|
||||
if item.uploaded_by_id is None:
|
||||
item.uploaded_by = temp.user
|
||||
update_fields.append("uploaded_by")
|
||||
if temp.e621_post_id is not None:
|
||||
item.e621_post_id = temp.e621_post_id
|
||||
item.e621_data = temp.e621_data
|
||||
update_fields += ["e621_post_id", "e621_data"]
|
||||
if temp.custom_rating in {"s", "q", "e"} and not item.rating:
|
||||
item.rating = temp.custom_rating
|
||||
update_fields.append("rating")
|
||||
if temp.custom_tags:
|
||||
item.tags = temp.custom_tags
|
||||
update_fields.append("tags")
|
||||
if temp.custom_notes:
|
||||
item.notes = temp.custom_notes
|
||||
update_fields.append("notes")
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
|
||||
temp.save()
|
||||
return item
|
||||
|
||||
|
||||
class TempUploadViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.RetrieveModelMixin,
|
||||
mixins.DestroyModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
serializer_class = TempUploadSerializer
|
||||
permission_classes = [CanUpload]
|
||||
parser_classes = [MultiPartParser, FormParser, JSONParser]
|
||||
http_method_names = ["get", "post", "delete", "head", "options"]
|
||||
|
||||
def get_queryset(self):
|
||||
queryset = TempUpload.objects.select_related("library_item")
|
||||
user = self.request.user
|
||||
if not (user.is_staff or user.is_superuser):
|
||||
queryset = queryset.filter(user=user)
|
||||
return queryset
|
||||
|
||||
def create(self, request):
|
||||
upload = request.FILES.get("file")
|
||||
if upload is None:
|
||||
return Response(
|
||||
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
extension = Path(upload.name).suffix.lower()
|
||||
if extension not in services.ALLOWED_EXTENSIONS:
|
||||
return Response(
|
||||
{"detail": f"Unsupported file type: {extension or 'unknown'}"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
temp = TempUpload.objects.create(
|
||||
user=request.user,
|
||||
file=upload,
|
||||
original_filename=upload.name,
|
||||
size=upload.size,
|
||||
)
|
||||
temp.md5 = services.compute_md5(temp.file.path)
|
||||
|
||||
existing = MediaItem.objects.filter(md5=temp.md5).first()
|
||||
if existing is not None:
|
||||
temp.status = TempUpload.STATUS_COMPLETED
|
||||
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
|
||||
temp.library_item = existing
|
||||
temp.file.delete(save=False)
|
||||
temp.save()
|
||||
return Response(
|
||||
TempUploadSerializer(temp).data, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["get"])
|
||||
def file(self, request, pk=None):
|
||||
temp = self.get_object()
|
||||
if not temp.file:
|
||||
raise Http404
|
||||
return services.serve_file(request, temp.file.path)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def iqdb(self, request, pk=None):
|
||||
temp = self.get_object()
|
||||
results = request.data.get("results")
|
||||
if not isinstance(results, list):
|
||||
return Response(
|
||||
{"detail": "results must be a list."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
temp.iqdb_data = services.sanitize_iqdb_results(results)
|
||||
if temp.status == TempUpload.STATUS_PENDING and temp.iqdb_data:
|
||||
temp.status = TempUpload.STATUS_VISUAL_MATCH
|
||||
temp.save(update_fields=["iqdb_data", "status", "updated_at"])
|
||||
return Response(TempUploadSerializer(temp).data)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def resolve(self, request, pk=None):
|
||||
temp = self.get_object()
|
||||
if temp.status == TempUpload.STATUS_COMPLETED:
|
||||
return Response(
|
||||
{"detail": "This upload is already in the library."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
mode = str(request.data.get("mode") or "").strip()
|
||||
if mode == "link":
|
||||
post = request.data.get("post")
|
||||
post_id = request.data.get("post_id")
|
||||
trimmed = services.trim_e621_post(post)
|
||||
has_numeric_id = post_id is not None and str(post_id).isdigit()
|
||||
if trimmed is None and not has_numeric_id:
|
||||
return Response(
|
||||
{"detail": "A post payload or post_id is required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if trimmed is not None and trimmed.get("id") is not None:
|
||||
temp.e621_post_id = int(trimmed["id"])
|
||||
elif has_numeric_id:
|
||||
temp.e621_post_id = int(post_id)
|
||||
temp.e621_data = trimmed
|
||||
temp.resolution = (
|
||||
TempUpload.RESOLUTION_AUTO_MD5
|
||||
if request.data.get("auto")
|
||||
else TempUpload.RESOLUTION_LINKED
|
||||
)
|
||||
rating = str(request.data.get("rating") or "")
|
||||
if rating in {"s", "q", "e"}:
|
||||
temp.custom_rating = rating
|
||||
tags = services.parse_tags(request.data.get("tags"))
|
||||
if tags is not None:
|
||||
temp.custom_tags = tags
|
||||
notes = request.data.get("notes")
|
||||
if notes is not None:
|
||||
temp.custom_notes = str(notes)
|
||||
elif mode == "custom":
|
||||
rating = str(request.data.get("rating") or "")
|
||||
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
|
||||
temp.custom_tags = services.parse_tags(request.data.get("tags")) or []
|
||||
temp.custom_notes = str(request.data.get("notes") or "")
|
||||
temp.resolution = TempUpload.RESOLUTION_CUSTOM
|
||||
else:
|
||||
return Response(
|
||||
{"detail": "mode must be 'link' or 'custom'."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
temp.save()
|
||||
try:
|
||||
complete_temp_upload(temp)
|
||||
except Exception as exc: # noqa: BLE001 - report completion failures
|
||||
logger.exception("Could not complete staged upload %s", temp.id)
|
||||
temp.status = TempUpload.STATUS_ERROR
|
||||
temp.save(update_fields=["status", "updated_at"])
|
||||
return Response(
|
||||
{"detail": f"Could not finish the upload: {exc}"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
temp.refresh_from_db()
|
||||
return Response(TempUploadSerializer(temp).data)
|
||||
|
||||
def perform_destroy(self, instance):
|
||||
if instance.file:
|
||||
instance.file.delete(save=False)
|
||||
instance.delete()
|
||||
@@ -1,14 +1,15 @@
|
||||
from django.urls import include, path
|
||||
from rest_framework.routers import DefaultRouter
|
||||
|
||||
from .views import DownloadToLibraryView, MediaItemViewSet, UploadView
|
||||
from .uploads import TempUploadViewSet
|
||||
from .views import DownloadToLibraryView, MediaItemViewSet
|
||||
|
||||
router = DefaultRouter()
|
||||
router.register("files", MediaItemViewSet, basename="file")
|
||||
router.register("uploads", TempUploadViewSet, basename="upload")
|
||||
|
||||
urlpatterns = [
|
||||
path("", include(router.urls)),
|
||||
path("uploads/", UploadView.as_view(), name="upload"),
|
||||
path(
|
||||
"online/downloads/",
|
||||
DownloadToLibraryView.as_view(),
|
||||
|
||||
@@ -8,7 +8,6 @@ from django.http import Http404
|
||||
from django.shortcuts import get_object_or_404
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.parsers import FormParser, MultiPartParser
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
@@ -165,55 +164,6 @@ class MediaItemViewSet(
|
||||
)
|
||||
|
||||
|
||||
class UploadView(APIView):
|
||||
permission_classes = [CanUpload]
|
||||
parser_classes = [MultiPartParser, FormParser]
|
||||
|
||||
def post(self, request):
|
||||
upload = request.FILES.get("file")
|
||||
if upload is None:
|
||||
return Response(
|
||||
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
destination = services.unique_destination(folder, upload.name)
|
||||
with open(destination, "wb") as output:
|
||||
for chunk in upload.chunks():
|
||||
output.write(chunk)
|
||||
try:
|
||||
item, created_item, _ = services.index_file(destination, folder)
|
||||
except Exception as exc: # noqa: BLE001 - report indexing failures
|
||||
destination.unlink(missing_ok=True)
|
||||
return Response(
|
||||
{"detail": f"Could not index upload: {exc}"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
update_fields = []
|
||||
if created_item and item.uploaded_by_id is None:
|
||||
item.uploaded_by = request.user
|
||||
update_fields.append("uploaded_by")
|
||||
rating = request.data.get("rating") or ""
|
||||
if rating in {"s", "q", "e"} and item.rating != rating:
|
||||
item.rating = rating
|
||||
update_fields.append("rating")
|
||||
tags = services.parse_tags(request.data.get("tags"))
|
||||
if tags is not None:
|
||||
item.tags = tags
|
||||
update_fields.append("tags")
|
||||
notes = request.data.get("notes")
|
||||
if notes is not None:
|
||||
item.notes = str(notes)
|
||||
update_fields.append("notes")
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
|
||||
return Response(
|
||||
MediaItemSerializer(item, context={"request": request}).data,
|
||||
status=status.HTTP_201_CREATED,
|
||||
)
|
||||
|
||||
|
||||
class DownloadToLibraryView(APIView):
|
||||
"""Fetch an e621 file into the watched folder and index it."""
|
||||
|
||||
|
||||
@@ -1,12 +1,32 @@
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { UploadCloud } from "lucide-react";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import {
|
||||
CheckCircle2,
|
||||
UploadCloud,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { useRef, useState, type DragEvent } from "react";
|
||||
import { Link } from "react-router-dom";
|
||||
|
||||
import { Button, EmptyState } from "@/components/ui";
|
||||
import { errorMessage, uploadFile } from "@/lib/api";
|
||||
import { Button, EmptyState, Spinner, inputClass } from "@/components/ui";
|
||||
import { api, authHeaders, errorMessage, uploadFile } from "@/lib/api";
|
||||
import { cn } from "@/lib/cn";
|
||||
import {
|
||||
E621Error,
|
||||
e621ErrorMessage,
|
||||
effectiveCredentials,
|
||||
fetchPost,
|
||||
fetchPostsByMd5,
|
||||
iqdbSearch,
|
||||
} from "@/lib/e621";
|
||||
import { formatBytes } from "@/lib/format";
|
||||
import type {
|
||||
E621IqdbCandidate,
|
||||
Paginated,
|
||||
Rating,
|
||||
TempUpload,
|
||||
} from "@/lib/types";
|
||||
import { useAuth } from "@/store/auth";
|
||||
import { useE621 } from "@/store/e621";
|
||||
|
||||
type UploadStatus = "queued" | "uploading" | "done" | "error";
|
||||
|
||||
@@ -18,13 +38,440 @@ interface UploadEntry {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const RESOLUTION_LABELS: Record<string, string> = {
|
||||
auto_md5: "auto-matched on e621",
|
||||
duplicate: "already in the library",
|
||||
linked: "linked to a post",
|
||||
custom: "custom metadata",
|
||||
};
|
||||
|
||||
const ratingChoices: { value: Rating; label: string; active: string }[] = [
|
||||
{
|
||||
value: "",
|
||||
label: "None",
|
||||
active: "border-ctp-surface2 bg-ctp-surface0 text-ctp-text",
|
||||
},
|
||||
{
|
||||
value: "s",
|
||||
label: "Safe",
|
||||
active: "border-ctp-green/40 bg-ctp-green/15 text-ctp-green",
|
||||
},
|
||||
{
|
||||
value: "q",
|
||||
label: "Questionable",
|
||||
active: "border-ctp-peach/40 bg-ctp-peach/15 text-ctp-peach",
|
||||
},
|
||||
{
|
||||
value: "e",
|
||||
label: "Explicit",
|
||||
active: "border-ctp-red/40 bg-ctp-red/15 text-ctp-red",
|
||||
},
|
||||
];
|
||||
|
||||
function describeError(error: unknown): string {
|
||||
if (error instanceof E621Error) return e621ErrorMessage(error);
|
||||
return errorMessage(error);
|
||||
}
|
||||
|
||||
async function fetchTempBlob(url: string): Promise<Blob> {
|
||||
const response = await fetch(url, { headers: authHeaders() });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Could not read the staged file (${response.status}).`);
|
||||
}
|
||||
return response.blob();
|
||||
}
|
||||
|
||||
function TempCard({
|
||||
temp,
|
||||
checking,
|
||||
onOpen,
|
||||
onCheck,
|
||||
onDismiss,
|
||||
}: {
|
||||
temp: TempUpload;
|
||||
checking: boolean;
|
||||
onOpen: () => void;
|
||||
onCheck: () => void;
|
||||
onDismiss: () => void;
|
||||
}) {
|
||||
const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename);
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
|
||||
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
|
||||
{temp.file_url ? (
|
||||
isVideo ? (
|
||||
<video
|
||||
src={temp.file_url}
|
||||
muted
|
||||
playsInline
|
||||
preload="metadata"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
) : (
|
||||
<img
|
||||
src={temp.file_url}
|
||||
alt={temp.original_filename}
|
||||
loading="lazy"
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
)
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
|
||||
{temp.status === "completed" ? "Indexed" : "No preview"}
|
||||
</div>
|
||||
)}
|
||||
{temp.status === "visual_match" && temp.iqdb_data?.length ? (
|
||||
<span className="absolute left-2 top-2 rounded-full bg-ctp-peach/90 px-1.5 py-0.5 font-mono text-[10px] font-medium text-ctp-crust">
|
||||
{temp.iqdb_data.length} similar
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<p
|
||||
className="mt-2 truncate font-mono text-xs text-ctp-subtext1"
|
||||
title={temp.original_filename}
|
||||
>
|
||||
{temp.original_filename}
|
||||
</p>
|
||||
<p className="font-mono text-[10px] text-ctp-overlay0">
|
||||
{formatBytes(temp.size)} · {temp.md5.slice(0, 10)}…
|
||||
</p>
|
||||
|
||||
{temp.status === "completed" && temp.library_j_id ? (
|
||||
<p className="mt-2 flex items-center gap-1.5 text-xs text-ctp-green">
|
||||
<CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
|
||||
<Link
|
||||
to={`/detail/${temp.library_j_id}`}
|
||||
className="font-mono hover:underline"
|
||||
>
|
||||
{temp.library_j_id}
|
||||
</Link>
|
||||
<span className="truncate text-ctp-overlay0">
|
||||
{RESOLUTION_LABELS[temp.resolution] ?? temp.resolution}
|
||||
</span>
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{temp.status === "error" ? (
|
||||
<p className="mt-2 text-xs text-ctp-red">
|
||||
Something went wrong while indexing this file.
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{temp.status === "pending" || temp.status === "visual_match" ? (
|
||||
<div className="mt-2 flex flex-wrap items-center gap-1.5">
|
||||
<Button className="px-2 py-1 text-xs" onClick={onOpen}>
|
||||
Set metadata
|
||||
</Button>
|
||||
{temp.status === "pending" ? (
|
||||
<Button
|
||||
variant="secondary"
|
||||
className="px-2 py-1 text-xs"
|
||||
onClick={onCheck}
|
||||
disabled={checking}
|
||||
>
|
||||
{checking ? "Checking…" : "Check similarity"}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={onDismiss}
|
||||
title={
|
||||
temp.status === "completed"
|
||||
? "Dismiss this record"
|
||||
: "Discard this staged upload"
|
||||
}
|
||||
className="mt-2 inline-flex items-center gap-1 rounded-md px-1.5 py-1 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
||||
>
|
||||
<X className="h-3 w-3" />
|
||||
{temp.status === "completed" ? "dismiss" : "discard"}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MetadataModal({
|
||||
temp,
|
||||
onClose,
|
||||
}: {
|
||||
temp: TempUpload;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const queryClient = useQueryClient();
|
||||
const credentials = useE621((state) => state.credentials);
|
||||
const [tab, setTab] = useState<"link" | "custom">("link");
|
||||
const [postId, setPostId] = useState("");
|
||||
const [rating, setRating] = useState<Rating>(temp.custom_rating);
|
||||
const [tags, setTags] = useState(temp.custom_tags.join(", "));
|
||||
const [notes, setNotes] = useState(temp.custom_notes);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename);
|
||||
|
||||
async function run(task: () => Promise<void>) {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
await task();
|
||||
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
||||
onClose();
|
||||
} catch (err) {
|
||||
setError(describeError(err));
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
function linkById() {
|
||||
const parsed = Number(postId.trim());
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) {
|
||||
setError("Enter a valid e621 post ID.");
|
||||
return;
|
||||
}
|
||||
return run(async () => {
|
||||
const post = await fetchPost(effectiveCredentials(credentials), parsed);
|
||||
await api(`/api/uploads/${temp.temp_id}/resolve/`, {
|
||||
method: "POST",
|
||||
json: { mode: "link", post, rating, tags, notes },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function linkCandidate(candidate: E621IqdbCandidate) {
|
||||
if (!candidate.post_id) return;
|
||||
return run(async () => {
|
||||
const post = await fetchPost(
|
||||
effectiveCredentials(credentials),
|
||||
candidate.post_id as number,
|
||||
);
|
||||
await api(`/api/uploads/${temp.temp_id}/resolve/`, {
|
||||
method: "POST",
|
||||
json: { mode: "link", post, rating, tags, notes },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function saveCustom() {
|
||||
return run(async () => {
|
||||
await api(`/api/uploads/${temp.temp_id}/resolve/`, {
|
||||
method: "POST",
|
||||
json: { mode: "custom", rating, tags, notes },
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className="fixed inset-0 z-[70] flex items-start justify-center overflow-y-auto bg-ctp-crust/70 px-4 py-10 backdrop-blur-sm"
|
||||
onClick={onClose}
|
||||
>
|
||||
<div
|
||||
className="w-full max-w-2xl rounded-lg border border-ctp-surface1 bg-ctp-base p-5"
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div className="min-w-0">
|
||||
<h2 className="text-sm font-semibold text-ctp-subtext1">
|
||||
Set metadata
|
||||
</h2>
|
||||
<p className="truncate font-mono text-xs text-ctp-overlay0">
|
||||
{temp.original_filename}
|
||||
</p>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
className="rounded-md p-1 text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
||||
title="Close"
|
||||
>
|
||||
<X className="h-4 w-4" />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="mt-4 grid gap-4 sm:grid-cols-[200px_minmax(0,1fr)]">
|
||||
<div className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle">
|
||||
{temp.file_url ? (
|
||||
isVideo ? (
|
||||
<video
|
||||
src={temp.file_url}
|
||||
controls
|
||||
muted
|
||||
className="max-h-52 w-full object-contain"
|
||||
/>
|
||||
) : (
|
||||
<img
|
||||
src={temp.file_url}
|
||||
alt={temp.original_filename}
|
||||
className="max-h-52 w-full object-contain"
|
||||
/>
|
||||
)
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="flex min-w-0 flex-col gap-3">
|
||||
<div className="flex gap-1.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTab("link")}
|
||||
className={cn(
|
||||
"rounded-md px-3 py-1.5 text-sm font-medium transition",
|
||||
tab === "link"
|
||||
? "bg-ctp-surface0 text-ctp-text"
|
||||
: "text-ctp-subtext0 hover:bg-ctp-surface0/60 hover:text-ctp-text",
|
||||
)}
|
||||
>
|
||||
Link to e621 post
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTab("custom")}
|
||||
className={cn(
|
||||
"rounded-md px-3 py-1.5 text-sm font-medium transition",
|
||||
tab === "custom"
|
||||
? "bg-ctp-surface0 text-ctp-text"
|
||||
: "text-ctp-subtext0 hover:bg-ctp-surface0/60 hover:text-ctp-text",
|
||||
)}
|
||||
>
|
||||
Custom metadata
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{tab === "link" ? (
|
||||
<div className="flex flex-col gap-3">
|
||||
{temp.iqdb_data && temp.iqdb_data.length > 0 ? (
|
||||
<div>
|
||||
<p className="text-xs font-medium uppercase tracking-wide text-ctp-overlay1">
|
||||
IQDB candidates
|
||||
</p>
|
||||
<div className="mt-2 grid grid-cols-3 gap-2">
|
||||
{temp.iqdb_data.map((candidate) => (
|
||||
<button
|
||||
key={`${candidate.post_id}-${candidate.md5}`}
|
||||
type="button"
|
||||
disabled={busy || !candidate.post_id}
|
||||
onClick={() => void linkCandidate(candidate)}
|
||||
className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle text-left transition hover:border-ctp-surface2 disabled:opacity-50"
|
||||
>
|
||||
{candidate.preview_url ? (
|
||||
<img
|
||||
src={candidate.preview_url}
|
||||
alt={`Post #${candidate.post_id}`}
|
||||
loading="lazy"
|
||||
className="aspect-square w-full object-cover"
|
||||
/>
|
||||
) : null}
|
||||
<span className="block px-1.5 py-1 font-mono text-[10px] text-ctp-overlay0">
|
||||
#{candidate.post_id}
|
||||
{candidate.score !== null
|
||||
? ` · ${Math.round(candidate.score)}%`
|
||||
: ""}
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-xs text-ctp-overlay0">
|
||||
No IQDB candidates yet — run "Check similarity" from the
|
||||
card, or enter a post ID below.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
className={cn(inputClass, "font-mono")}
|
||||
placeholder="e621 post ID"
|
||||
value={postId}
|
||||
onChange={(event) => setPostId(event.target.value)}
|
||||
/>
|
||||
<Button onClick={() => void linkById()} disabled={busy}>
|
||||
Link
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{ratingChoices.map((choice) => (
|
||||
<button
|
||||
key={choice.value || "none"}
|
||||
type="button"
|
||||
onClick={() => setRating(choice.value)}
|
||||
className={cn(
|
||||
"rounded-full border px-2.5 py-1 text-xs font-medium transition",
|
||||
rating === choice.value
|
||||
? choice.active
|
||||
: "border-ctp-surface1 text-ctp-subtext0 hover:border-ctp-surface2 hover:text-ctp-text",
|
||||
)}
|
||||
>
|
||||
{choice.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<input
|
||||
className={inputClass}
|
||||
placeholder="Tags, comma separated"
|
||||
value={tags}
|
||||
onChange={(event) => setTags(event.target.value)}
|
||||
/>
|
||||
<textarea
|
||||
className={cn(inputClass, "min-h-16 resize-y")}
|
||||
placeholder="Notes…"
|
||||
value={notes}
|
||||
onChange={(event) => setNotes(event.target.value)}
|
||||
/>
|
||||
<Button onClick={() => void saveCustom()} disabled={busy}>
|
||||
Save custom metadata
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{error ? (
|
||||
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
{busy ? (
|
||||
<p className="flex items-center gap-2 text-xs text-ctp-overlay0">
|
||||
<Spinner className="h-3.5 w-3.5" /> Working…
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function UploadPage() {
|
||||
const queryClient = useQueryClient();
|
||||
const user = useAuth((state) => state.user);
|
||||
const credentials = useE621((state) => state.credentials);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const [entries, setEntries] = useState<UploadEntry[]>([]);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [processing, setProcessing] = useState(false);
|
||||
const [checkingIds, setCheckingIds] = useState<string[]>([]);
|
||||
const [modalFor, setModalFor] = useState<TempUpload | null>(null);
|
||||
|
||||
const uploadsQuery = useQuery({
|
||||
queryKey: ["uploads"],
|
||||
queryFn: () => api<Paginated<TempUpload>>("/api/uploads/"),
|
||||
enabled: Boolean(user?.can_upload),
|
||||
});
|
||||
const uploads = uploadsQuery.data?.results ?? [];
|
||||
|
||||
const pending = uploads.filter((temp) => temp.status === "pending");
|
||||
const visual = uploads.filter((temp) => temp.status === "visual_match");
|
||||
const completed = uploads.filter(
|
||||
(temp) => temp.status === "completed" || temp.status === "error",
|
||||
);
|
||||
|
||||
const pendingCount = entries.filter(
|
||||
(entry) => entry.status === "queued" || entry.status === "error",
|
||||
@@ -55,19 +502,115 @@ export default function UploadPage() {
|
||||
);
|
||||
}
|
||||
|
||||
function invalidateUploads() {
|
||||
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
|
||||
}
|
||||
|
||||
async function refreshUploads(): Promise<TempUpload[]> {
|
||||
const fresh = await api<Paginated<TempUpload>>("/api/uploads/");
|
||||
queryClient.setQueryData(["uploads"], fresh);
|
||||
return fresh.results;
|
||||
}
|
||||
|
||||
async function checkSimilarity(ids: string[]) {
|
||||
if (ids.length === 0) return;
|
||||
setProcessing(true);
|
||||
setCheckingIds((current) => [...current, ...ids]);
|
||||
try {
|
||||
const fresh = await refreshUploads();
|
||||
const targets = fresh.filter(
|
||||
(temp) => ids.includes(temp.temp_id) && temp.status === "pending",
|
||||
);
|
||||
for (const temp of targets) {
|
||||
if (!temp.file_url) continue;
|
||||
try {
|
||||
const blob = await fetchTempBlob(temp.file_url);
|
||||
const results = await iqdbSearch(
|
||||
effectiveCredentials(credentials),
|
||||
blob,
|
||||
temp.original_filename,
|
||||
);
|
||||
if (results.length > 0) {
|
||||
await api(`/api/uploads/${temp.temp_id}/iqdb/`, {
|
||||
method: "POST",
|
||||
json: { results },
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Skip this file; the card stays pending.
|
||||
} finally {
|
||||
setCheckingIds((current) =>
|
||||
current.filter((value) => value !== temp.temp_id),
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
setProcessing(false);
|
||||
setCheckingIds([]);
|
||||
invalidateUploads();
|
||||
}
|
||||
}
|
||||
|
||||
async function autoMatchAndCheck(created: TempUpload[]) {
|
||||
const creds = effectiveCredentials(credentials);
|
||||
const unresolved = created.filter((temp) => temp.status === "pending");
|
||||
|
||||
// 1) Auto-complete files whose MD5 matches an e621 post.
|
||||
if (unresolved.length > 0) {
|
||||
try {
|
||||
for (let index = 0; index < unresolved.length; index += 20) {
|
||||
const batch = unresolved.slice(index, index + 20);
|
||||
const posts = await fetchPostsByMd5(
|
||||
creds,
|
||||
batch.map((temp) => temp.md5),
|
||||
);
|
||||
for (const post of posts) {
|
||||
const match = batch.find((temp) => temp.md5 === post.file.md5);
|
||||
if (!match) continue;
|
||||
try {
|
||||
await api(`/api/uploads/${match.temp_id}/resolve/`, {
|
||||
method: "POST",
|
||||
json: { mode: "link", post, auto: true },
|
||||
});
|
||||
} catch {
|
||||
// Leave it pending.
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// e621 unavailable; leave everything pending.
|
||||
}
|
||||
invalidateUploads();
|
||||
}
|
||||
|
||||
// 2) IQDB similarity for whatever is still pending.
|
||||
const fresh = await refreshUploads();
|
||||
const stillPending = fresh
|
||||
.filter((temp) => temp.status === "pending")
|
||||
.map((temp) => temp.temp_id);
|
||||
void checkSimilarity(stillPending);
|
||||
}
|
||||
|
||||
async function startUploads() {
|
||||
if (busy) return;
|
||||
setBusy(true);
|
||||
setProcessing(true);
|
||||
const queue = entries.filter(
|
||||
(entry) => entry.status === "queued" || entry.status === "error",
|
||||
);
|
||||
const created: TempUpload[] = [];
|
||||
for (const entry of queue) {
|
||||
updateEntry(entry.id, { status: "uploading", progress: 0, error: undefined });
|
||||
updateEntry(entry.id, {
|
||||
status: "uploading",
|
||||
progress: 0,
|
||||
error: undefined,
|
||||
});
|
||||
try {
|
||||
await uploadFile(entry.file, {}, (percent) =>
|
||||
const temp = await uploadFile(entry.file, {}, (percent) =>
|
||||
updateEntry(entry.id, { progress: percent }),
|
||||
);
|
||||
updateEntry(entry.id, { status: "done", progress: 100 });
|
||||
created.push(temp);
|
||||
} catch (error) {
|
||||
updateEntry(entry.id, {
|
||||
status: "error",
|
||||
@@ -75,8 +618,15 @@ export default function UploadPage() {
|
||||
});
|
||||
}
|
||||
}
|
||||
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
||||
setBusy(false);
|
||||
invalidateUploads();
|
||||
void autoMatchAndCheck(created).finally(() => setProcessing(false));
|
||||
}
|
||||
|
||||
function dismiss(temp: TempUpload) {
|
||||
void api(`/api/uploads/${temp.temp_id}/`, { method: "DELETE" }).then(() =>
|
||||
invalidateUploads(),
|
||||
);
|
||||
}
|
||||
|
||||
function clearFinished() {
|
||||
@@ -97,13 +647,54 @@ export default function UploadPage() {
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mx-auto flex w-full max-w-3xl flex-col gap-6">
|
||||
<header>
|
||||
<h1 className="text-lg font-semibold">Upload</h1>
|
||||
<p className="mt-1 text-sm text-ctp-overlay0">
|
||||
Files are written to the watched folder and indexed into the library.
|
||||
const column = (
|
||||
title: string,
|
||||
items: TempUpload[],
|
||||
emptyText: string,
|
||||
) => (
|
||||
<section className="flex min-w-0 flex-col gap-3">
|
||||
<header className="flex items-center justify-between gap-2">
|
||||
<h2 className="text-sm font-semibold text-ctp-subtext1">{title}</h2>
|
||||
<span className="font-mono text-[11px] text-ctp-overlay0">
|
||||
{items.length}
|
||||
</span>
|
||||
</header>
|
||||
{items.length === 0 ? (
|
||||
<p className="rounded-lg border border-dashed border-ctp-surface1 px-3 py-6 text-center text-xs text-ctp-overlay0">
|
||||
{emptyText}
|
||||
</p>
|
||||
) : (
|
||||
<div className="flex flex-col gap-3">
|
||||
{items.map((temp) => (
|
||||
<TempCard
|
||||
key={temp.temp_id}
|
||||
temp={temp}
|
||||
checking={checkingIds.includes(temp.temp_id)}
|
||||
onOpen={() => setModalFor(temp)}
|
||||
onCheck={() => void checkSimilarity([temp.temp_id])}
|
||||
onDismiss={() => dismiss(temp)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="mx-auto flex w-full max-w-6xl flex-col gap-6">
|
||||
<header className="flex flex-wrap items-end justify-between gap-3">
|
||||
<div>
|
||||
<h1 className="text-lg font-semibold">Upload</h1>
|
||||
<p className="mt-1 text-sm text-ctp-overlay0">
|
||||
Files are staged first, auto-matched by MD5, checked against IQDB,
|
||||
then moved into the library once resolved.
|
||||
</p>
|
||||
</div>
|
||||
{processing ? (
|
||||
<p className="flex items-center gap-2 text-xs text-ctp-overlay0">
|
||||
<Spinner className="h-3.5 w-3.5" /> Processing uploads…
|
||||
</p>
|
||||
) : null}
|
||||
</header>
|
||||
|
||||
<div
|
||||
@@ -122,13 +713,13 @@ export default function UploadPage() {
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={handleDrop}
|
||||
className={cn(
|
||||
"flex cursor-pointer flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed px-6 py-14 text-center transition",
|
||||
"flex cursor-pointer flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed px-6 py-10 text-center transition",
|
||||
dragging
|
||||
? "border-ctp-mauve bg-ctp-mauve/5"
|
||||
: "border-ctp-surface1 hover:border-ctp-surface2",
|
||||
)}
|
||||
>
|
||||
<UploadCloud className="h-8 w-8 text-ctp-overlay0" />
|
||||
<UploadCloud className="h-7 w-7 text-ctp-overlay0" />
|
||||
<p className="text-sm font-medium text-ctp-subtext1">
|
||||
Drag & drop files here
|
||||
</p>
|
||||
@@ -148,7 +739,7 @@ export default function UploadPage() {
|
||||
</div>
|
||||
|
||||
{entries.length > 0 ? (
|
||||
<section className="flex flex-col gap-3">
|
||||
<section className="flex flex-col gap-2">
|
||||
{entries.map((entry) => (
|
||||
<div
|
||||
key={entry.id}
|
||||
@@ -177,35 +768,46 @@ export default function UploadPage() {
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="mt-1.5 flex items-center justify-between text-xs">
|
||||
<span className="text-ctp-overlay0">
|
||||
{entry.status === "queued"
|
||||
? "Queued"
|
||||
: entry.status === "uploading"
|
||||
? `Uploading… ${entry.progress}%`
|
||||
: entry.status === "done"
|
||||
? "Done"
|
||||
: (entry.error ?? "Failed")}
|
||||
</span>
|
||||
<div className="mt-1.5 text-xs text-ctp-overlay0">
|
||||
{entry.status === "queued"
|
||||
? "Queued"
|
||||
: entry.status === "uploading"
|
||||
? `Uploading… ${entry.progress}%`
|
||||
: entry.status === "done"
|
||||
? "Uploaded — processing"
|
||||
: (entry.error ?? "Failed")}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div className="flex items-center gap-3">
|
||||
<Button
|
||||
onClick={() => void startUploads()}
|
||||
disabled={busy || pendingCount === 0}
|
||||
>
|
||||
{busy
|
||||
? "Uploading…"
|
||||
: `Upload ${pendingCount || ""} file${pendingCount === 1 ? "" : "s"}`.trim()}
|
||||
</Button>
|
||||
<Button variant="ghost" onClick={clearFinished} disabled={busy}>
|
||||
Clear finished
|
||||
</Button>
|
||||
</div>
|
||||
</section>
|
||||
) : (
|
||||
<EmptyState
|
||||
title="No files selected"
|
||||
description="Drop files above to queue them for upload."
|
||||
/>
|
||||
)}
|
||||
) : null}
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<Button onClick={() => void startUploads()} disabled={busy || pendingCount === 0}>
|
||||
{busy ? "Uploading…" : `Upload ${pendingCount || ""} file${pendingCount === 1 ? "" : "s"}`.trim()}
|
||||
</Button>
|
||||
<Button variant="ghost" onClick={clearFinished} disabled={busy}>
|
||||
Clear finished
|
||||
</Button>
|
||||
<div className="grid gap-6 lg:grid-cols-3">
|
||||
{column("Pending & Unmatched", pending, "Nothing waiting.")}
|
||||
{column(
|
||||
"Visual Similarity Detected",
|
||||
visual,
|
||||
"No IQDB matches right now.",
|
||||
)}
|
||||
{column("Auto-uploaded & Indexed", completed, "Nothing indexed yet.")}
|
||||
</div>
|
||||
|
||||
{modalFor ? (
|
||||
<MetadataModal temp={modalFor} onClose={() => setModalFor(null)} />
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
+10
-3
@@ -1,4 +1,4 @@
|
||||
import type { MediaItem } from "@/lib/types";
|
||||
import type { TempUpload } from "@/lib/types";
|
||||
|
||||
const TOKEN_KEY = "j621.token";
|
||||
|
||||
@@ -6,6 +6,13 @@ export function getToken(): string | null {
|
||||
return window.localStorage.getItem(TOKEN_KEY);
|
||||
}
|
||||
|
||||
export function authHeaders(extra?: HeadersInit): Headers {
|
||||
const headers = new Headers(extra);
|
||||
const token = getToken();
|
||||
if (token) headers.set("Authorization", `Token ${token}`);
|
||||
return headers;
|
||||
}
|
||||
|
||||
export function setToken(token: string | null): void {
|
||||
if (token) {
|
||||
window.localStorage.setItem(TOKEN_KEY, token);
|
||||
@@ -82,7 +89,7 @@ export function uploadFile(
|
||||
file: File,
|
||||
fields: Record<string, string> = {},
|
||||
onProgress?: (percent: number) => void,
|
||||
): Promise<MediaItem> {
|
||||
): Promise<TempUpload> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
@@ -107,7 +114,7 @@ export function uploadFile(
|
||||
data = null;
|
||||
}
|
||||
if (request.status >= 200 && request.status < 300) {
|
||||
resolve(data as MediaItem);
|
||||
resolve(data as TempUpload);
|
||||
} else {
|
||||
reject(new ApiError(request.status, data));
|
||||
}
|
||||
|
||||
@@ -296,6 +296,90 @@ export async function setFavorite(
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchPostsByMd5(
|
||||
credentials: E621Credentials,
|
||||
md5s: string[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<E621Post[]> {
|
||||
if (md5s.length === 0) return [];
|
||||
const data = await e621Request<{ posts: E621Post[] }>(
|
||||
credentials,
|
||||
"/posts.json",
|
||||
{
|
||||
params: {
|
||||
tags: `md5:${md5s.join(",")}`,
|
||||
limit: Math.min(md5s.length, 100),
|
||||
},
|
||||
signal,
|
||||
},
|
||||
);
|
||||
return data.posts;
|
||||
}
|
||||
|
||||
export interface E621IqdbResult {
|
||||
post_id: number | null;
|
||||
score: number | null;
|
||||
preview_url: string | null;
|
||||
rating: string | null;
|
||||
md5: string | null;
|
||||
}
|
||||
|
||||
function normalizeIqdbEntry(entry: unknown): E621IqdbResult | null {
|
||||
if (!entry || typeof entry !== "object") return null;
|
||||
const record = entry as Record<string, unknown>;
|
||||
const postWrap = record.post;
|
||||
let post: Record<string, unknown> = {};
|
||||
if (postWrap && typeof postWrap === "object") {
|
||||
const wrap = postWrap as Record<string, unknown>;
|
||||
post =
|
||||
wrap.posts && typeof wrap.posts === "object"
|
||||
? (wrap.posts as Record<string, unknown>)
|
||||
: wrap;
|
||||
}
|
||||
const file = (post.file ?? {}) as Record<string, unknown>;
|
||||
const preview = (post.preview ?? {}) as Record<string, unknown>;
|
||||
const postId =
|
||||
typeof record.post_id === "number"
|
||||
? record.post_id
|
||||
: typeof post.id === "number"
|
||||
? post.id
|
||||
: null;
|
||||
return {
|
||||
post_id: postId,
|
||||
score: typeof record.score === "number" ? record.score : null,
|
||||
preview_url: typeof preview.url === "string" ? preview.url : null,
|
||||
rating: typeof post.rating === "string" ? post.rating : null,
|
||||
md5: typeof file.md5 === "string" ? file.md5 : null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function iqdbSearch(
|
||||
credentials: E621Credentials,
|
||||
file: Blob,
|
||||
filename: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<E621IqdbResult[]> {
|
||||
const form = new FormData();
|
||||
form.append("file", file, filename);
|
||||
const data = await e621Request<unknown>(credentials, "/iqdb_queries.json", {
|
||||
method: "POST",
|
||||
body: form,
|
||||
signal,
|
||||
});
|
||||
const list = Array.isArray(data)
|
||||
? data
|
||||
: Array.isArray((data as { matches?: unknown[] })?.matches)
|
||||
? (data as { matches: unknown[] }).matches
|
||||
: [];
|
||||
return list
|
||||
.map(normalizeIqdbEntry)
|
||||
.filter(
|
||||
(entry): entry is E621IqdbResult =>
|
||||
entry !== null && (entry.post_id !== null || entry.preview_url !== null),
|
||||
)
|
||||
.slice(0, 10);
|
||||
}
|
||||
|
||||
export function postIsBlacklisted(
|
||||
post: E621Post,
|
||||
blacklist: Set<string>,
|
||||
|
||||
@@ -106,3 +106,30 @@ export interface FileNeighbors {
|
||||
previous: FileNeighbor | null;
|
||||
next: FileNeighbor | null;
|
||||
}
|
||||
|
||||
export interface E621IqdbCandidate {
|
||||
post_id: number | null;
|
||||
score: number | null;
|
||||
preview_url: string | null;
|
||||
rating: string | null;
|
||||
md5: string | null;
|
||||
}
|
||||
|
||||
export interface TempUpload {
|
||||
temp_id: string;
|
||||
original_filename: string;
|
||||
md5: string;
|
||||
size: number;
|
||||
status: "pending" | "visual_match" | "completed" | "error";
|
||||
resolution: "" | "auto_md5" | "duplicate" | "linked" | "custom";
|
||||
e621_post_id: number | null;
|
||||
e621_data: E621StoredPost | null;
|
||||
custom_rating: Rating;
|
||||
custom_tags: string[];
|
||||
custom_notes: string;
|
||||
iqdb_data: E621IqdbCandidate[] | null;
|
||||
library_j_id: string | null;
|
||||
file_url: string | null;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user