From 39307cb141ab6dbb7433f310dbf402444fef5e7c Mon Sep 17 00:00:00 2001 From: JakeBreath Date: Fri, 18 Sep 2026 19:58:17 -0500 Subject: [PATCH] Unpaginate staged uploads and make big upload batches visible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upload board partitions /api/uploads/ into Pending / Visual similarity / Auto-uploaded, but the endpoint was paginated at 48 — a 69-file batch silently lost 21 entries, and the similarity sweep (which reads the same list back after uploading) only ever saw the first page. The staged-upload list is now unpaginated: it is a transient per-user set, still limited to the caller's rows and the uploader role. The page takes a plain array. Watching progress with dozens of files was also poor: - the queue uploads three files at a time instead of strictly one at a time; - the Uploads section now shows a batch bar and 'n/m uploaded · x%' next to the count, so the overall progress never scrolls out of sight; - entries are ordered active-first (uploading, queued, failed, done) so the file being uploaded is always at the top of the grid; - tiles are larger (4 columns at lg instead of 5); - the header reads 'Uploading n/m…' and 'Checking n file(s) against IQDB…' instead of a bare spinner. Tests: staged-upload list unpaginated past 48, per-user, uploader-only (3 new; full suite 49 green). Live-checked the bare-array response. --- backend/apps/library/tests/test_uploads.py | 75 ++++++++++++ backend/apps/library/uploads.py | 3 + frontend/src/features/upload/UploadPage.tsx | 122 +++++++++++++++----- 3 files changed, 168 insertions(+), 32 deletions(-) create mode 100644 backend/apps/library/tests/test_uploads.py diff --git a/backend/apps/library/tests/test_uploads.py b/backend/apps/library/tests/test_uploads.py new file mode 100644 index 0000000..4869235 --- /dev/null +++ b/backend/apps/library/tests/test_uploads.py @@ -0,0 +1,75 @@ +"""The staged-upload list is a complete set, not a page. + +The upload board partitions it into Pending / Visual similarity / Auto-uploaded +columns; with the standard 48-item pagination a 69-file batch silently lost 21 +entries from the board (and from the similarity sweep that reads it back). +""" + +import hashlib + +from django.contrib.auth import get_user_model +from django.test import Client, TestCase + +from rest_framework.authtoken.models import Token + +from apps.library.models import TempUpload + +User = get_user_model() + + +class TempUploadListTests(TestCase): + def setUp(self): + self.uploader = User.objects.create_user( + username="upload-user", password="upload-pass-123456" + ) + self.uploader.role = "uploader" + self.uploader.save(update_fields=["role"]) + self.other = User.objects.create_user( + username="upload-other", password="upload-pass-123456" + ) + self.other.role = "uploader" + self.other.save(update_fields=["role"]) + + def api_client(self, user): + client = Client() + client.defaults["HTTP_AUTHORIZATION"] = ( + f"Token {Token.objects.create(user=user).key}" + ) + return client + + def test_list_is_not_paginated_past_48(self): + TempUpload.objects.bulk_create( + [ + TempUpload( + user=self.uploader, + original_filename=f"file_{index:03d}.png", + md5=hashlib.md5(f"file-{index}".encode()).hexdigest(), + size=index, + ) + for index in range(69) + ] + ) + response = self.api_client(self.uploader).get("/api/uploads/") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIsInstance(data, list, "the list must not be paginated") + self.assertEqual(len(data), 69) + + def test_list_only_contains_the_callers_uploads(self): + TempUpload.objects.create( + user=self.uploader, + original_filename="mine.png", + md5=hashlib.md5(b"mine").hexdigest(), + size=1, + ) + TempUpload.objects.create( + user=self.other, + original_filename="theirs.png", + md5=hashlib.md5(b"theirs").hexdigest(), + size=1, + ) + mine = self.api_client(self.uploader).get("/api/uploads/").json() + self.assertEqual([row["original_filename"] for row in mine], ["mine.png"]) + + def test_anonymous_cannot_list(self): + self.assertEqual(Client().get("/api/uploads/").status_code, 401) diff --git a/backend/apps/library/uploads.py b/backend/apps/library/uploads.py index 8a63938..63c0899 100644 --- a/backend/apps/library/uploads.py +++ b/backend/apps/library/uploads.py @@ -150,6 +150,9 @@ class TempUploadViewSet( serializer_class = TempUploadSerializer permission_classes = [CanUpload] parser_classes = [MultiPartParser, FormParser, JSONParser] + # Unpaginated: the board shows every staged upload (69-file batches were + # silently cut to the API's 48-item page). + pagination_class = None http_method_names = ["get", "post", "delete", "head", "options"] def get_queryset(self): diff --git a/frontend/src/features/upload/UploadPage.tsx b/frontend/src/features/upload/UploadPage.tsx index 10358fc..87ca2e2 100644 --- a/frontend/src/features/upload/UploadPage.tsx +++ b/frontend/src/features/upload/UploadPage.tsx @@ -7,6 +7,7 @@ import { } from "lucide-react"; import { useEffect, + useMemo, useRef, useState, type DragEvent, @@ -27,12 +28,7 @@ import { RATING_LABELS, } from "@/lib/e621"; import { formatBytes } from "@/lib/format"; -import type { - E621IqdbCandidate, - Paginated, - Rating, - TempUpload, -} from "@/lib/types"; +import type { E621IqdbCandidate, Rating, TempUpload } from "@/lib/types"; import { useAuth } from "@/store/auth"; import { confirmAction } from "@/store/confirm"; import { useE621 } from "@/store/e621"; @@ -40,6 +36,17 @@ import { toast } from "@/store/toasts"; type UploadStatus = "queued" | "uploading" | "done" | "error"; +/** How many files upload at once; big batches were painfully serial. */ +const UPLOAD_CONCURRENCY = 3; + +/** Display order: the file being uploaded stays at the top of the grid. */ +const STATUS_ORDER: Record = { + uploading: 0, + queued: 1, + error: 2, + done: 3, +}; + interface UploadEntry { id: string; file: File; @@ -642,10 +649,11 @@ export default function UploadPage() { const uploadsQuery = useQuery({ queryKey: ["uploads"], - queryFn: () => api>("/api/uploads/"), + // Unpaginated: the board partitions the complete staging list. + queryFn: () => api("/api/uploads/"), enabled: Boolean(user?.can_upload), }); - const uploads = uploadsQuery.data?.results ?? []; + const uploads = uploadsQuery.data ?? []; const pending = uploads.filter((temp) => temp.status === "pending"); const visual = uploads.filter((temp) => temp.status === "visual_match"); @@ -653,6 +661,32 @@ export default function UploadPage() { (temp) => temp.status === "completed" || temp.status === "error", ); + // Active work first so a long batch never hides the file in progress. + const orderedEntries = useMemo( + () => + [...entries].sort( + (a, b) => STATUS_ORDER[a.status] - STATUS_ORDER[b.status], + ), + [entries], + ); + + const uploadedCount = entries.filter((entry) => entry.status === "done").length; + const failedCount = entries.filter((entry) => entry.status === "error").length; + const batchPercent = entries.length + ? Math.round( + entries.reduce( + (total, entry) => + total + + (entry.status === "done" + ? 100 + : entry.status === "queued" + ? 0 + : entry.progress), + 0, + ) / entries.length, + ) + : 0; + function addFiles(files: FileList | null) { if (!files || files.length === 0) return; const next: UploadEntry[] = Array.from(files).map((file) => { @@ -691,9 +725,9 @@ export default function UploadPage() { } async function refreshUploads(): Promise { - const fresh = await api>("/api/uploads/"); + const fresh = await api("/api/uploads/"); queryClient.setQueryData(["uploads"], fresh); - return fresh.results; + return fresh; } async function checkSimilarity(ids: string[]) { @@ -780,27 +814,35 @@ export default function UploadPage() { runningRef.current = true; setBusy(true); const created: TempUpload[] = []; - while (queueRef.current.length > 0) { - const entry = queueRef.current.shift(); - if (!entry) break; - updateEntry(entry.id, { - status: "uploading", - progress: 0, - error: undefined, - }); - try { - const temp = await uploadFile(entry.file, {}, (percent) => - updateEntry(entry.id, { progress: percent }), - ); - updateEntry(entry.id, { status: "done", progress: 100 }); - created.push(temp); - } catch (error) { + + async function worker() { + while (queueRef.current.length > 0) { + const entry = queueRef.current.shift(); + if (!entry) return; updateEntry(entry.id, { - status: "error", - error: errorMessage(error), + status: "uploading", + progress: 0, + error: undefined, }); + try { + const temp = await uploadFile(entry.file, {}, (percent) => + updateEntry(entry.id, { progress: percent }), + ); + updateEntry(entry.id, { status: "done", progress: 100 }); + created.push(temp); + } catch (error) { + updateEntry(entry.id, { + status: "error", + error: errorMessage(error), + }); + } } } + + await Promise.all( + Array.from({ length: UPLOAD_CONCURRENCY }, () => worker()), + ); + runningRef.current = false; setBusy(false); invalidateUploads(); @@ -899,7 +941,11 @@ export default function UploadPage() { {busy || processing ? (

- {busy ? "Uploading…" : "Processing uploads…"} + {busy + ? `Uploading ${Math.min(uploadedCount + 1, entries.length)}/${entries.length}…` + : checkingIds.length > 0 + ? `Checking ${checkingIds.length} file(s) against IQDB…` + : "Processing uploads…"}

) : null} @@ -950,11 +996,12 @@ export default function UploadPage() { {entries.length > 0 ? (
-
+

Uploads - {entries.length} + {uploadedCount}/{entries.length} uploaded + {failedCount > 0 ? ` · ${failedCount} failed` : ""}

-
- {entries.map((entry) => ( +
+
+
+
+ + {batchPercent}% + +
+
+ {orderedEntries.map((entry) => (