From e62d7af42fa68513ce41e658b9027eafcf9d812c Mon Sep 17 00:00:00 2001 From: JakeBreath Date: Sat, 19 Sep 2026 01:18:50 -0500 Subject: [PATCH] Live-updating upload board, self-clearing tiles, original MD5 batch size - MD5 auto-match now sends 75 md5: metatags per posts.json query, the same batch size the original J621-Django app used (was 20); the limit cap no longer truncates batches. - every settled upload is upserted into the staged list right away, so the Pending / Visual Similarity / Auto-uploaded columns move as files land instead of waiting for the whole batch (auto-matched resolves and IQDB results use the same path). - finished upload tiles fade out and remove themselves ~2s after completing; failures stay until cleared. Batch counters are tracked separately from the visible tiles so the header and progress bar stay accurate as tiles vanish. - AGENTS.md now points at the original Django app for reference behavior. Verified live with a headless upload run: the columns showed the new files immediately and the 8 tiles were gone ~3s after finishing. --- AGENTS.md | 5 + frontend/src/features/upload/UploadPage.tsx | 117 ++++++++++++++------ frontend/src/lib/e621.ts | 2 +- 3 files changed, 88 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a56ae9b..18c81ed 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,11 @@ Use the venv on backend/venv/ DO NOT mess with the system's python +The original J621 (Django) app is kept for reference at + /mnt/Disco/Proyects/Python3.13/J621-Django/ +Consult it when matching original behavior (batch sizes for e621 lookups, +per-page sets, upload flow) instead of guessing. + For Frontend work: read both Frontend Design related Markdown files on the frontend/ folder diff --git a/frontend/src/features/upload/UploadPage.tsx b/frontend/src/features/upload/UploadPage.tsx index 4591831..86ee394 100644 --- a/frontend/src/features/upload/UploadPage.tsx +++ b/frontend/src/features/upload/UploadPage.tsx @@ -41,12 +41,19 @@ type UploadStatus = "queued" | "uploading" | "done" | "error"; /** How many files upload at once; big batches were painfully serial. */ const UPLOAD_CONCURRENCY = 3; +/** md5: metatags per posts.json query — same batch size as the original app. */ +const MD5_BATCH_SIZE = 75; + +/** Finished upload tiles clear themselves this long after completing. */ +const ENTRY_CLEAR_MS = 2000; + interface UploadEntry { id: string; file: File; progress: number; status: UploadStatus; previewUrl: string; + fading?: boolean; error?: string; } @@ -700,6 +707,7 @@ export default function UploadPage() { const objectUrls = useRef([]); const [dragging, setDragging] = useState(false); const [entries, setEntries] = useState([]); + const [batch, setBatch] = useState({ total: 0, done: 0, failed: 0 }); const [busy, setBusy] = useState(false); const [processing, setProcessing] = useState(false); const [checkingIds, setCheckingIds] = useState([]); @@ -746,20 +754,18 @@ export default function UploadPage() { (entry) => entry.status === "uploading", ); - const uploadedCount = entries.filter((entry) => entry.status === "done").length; - const failedCount = entries.filter((entry) => entry.status === "error").length; - const batchPercent = entries.length - ? Math.round( - entries.reduce( - (total, entry) => - total + - (entry.status === "done" - ? 100 - : entry.status === "queued" - ? 0 - : entry.progress), - 0, - ) / entries.length, + // Finished tiles clear themselves after a moment, so the counters come + // from batch totals rather than the tiles currently on screen. + const settledCount = batch.done + batch.failed; + const inflightProgress = entries.reduce( + (total, entry) => + total + (entry.status === "uploading" ? entry.progress : 0), + 0, + ); + const batchPercent = batch.total + ? Math.min( + 100, + Math.round((settledCount * 100 + inflightProgress) / batch.total), ) : 0; @@ -780,6 +786,15 @@ export default function UploadPage() { }; }); setEntries((current) => [...current, ...next]); + // Start a fresh batch when the previous one is fully settled. + setBatch((current) => { + const settled = current.done + current.failed >= current.total; + return { + total: (settled ? 0 : current.total) + next.length, + done: settled ? 0 : current.done, + failed: settled ? 0 : current.failed, + }; + }); queueRef.current.push(...next); void drainQueue(); } @@ -800,6 +815,17 @@ export default function UploadPage() { void queryClient.invalidateQueries({ queryKey: ["uploads"] }); } + /** Patch one staged upload into the list so the board moves as work happens. */ + function upsertUpload(temp: TempUpload) { + queryClient.setQueryData(["uploads"], (current) => { + if (!current) return current; + // The API lists newest first, so unseen uploads go on top. + return current.some((item) => item.temp_id === temp.temp_id) + ? current.map((item) => (item.temp_id === temp.temp_id ? temp : item)) + : [temp, ...current]; + }); + } + async function refreshUploads(): Promise { const fresh = await api("/api/uploads/"); queryClient.setQueryData(["uploads"], fresh); @@ -870,13 +896,7 @@ export default function UploadPage() { `/api/uploads/${temp.temp_id}/iqdb/`, { method: "POST", json: { results } }, ); - queryClient.setQueryData(["uploads"], (current) => - current - ? current.map((item) => - item.temp_id === updated.temp_id ? updated : item, - ) - : current, - ); + upsertUpload(updated); } catch (error) { setCheckErrors((current) => ({ ...current, @@ -915,23 +935,29 @@ export default function UploadPage() { const creds = effectiveCredentials(credentials); const unresolved = created.filter((temp) => temp.status === "pending"); - // 1) Auto-complete files whose MD5 matches an e621 post. + // 1) Auto-complete files whose MD5 matches an e621 post. The original app + // sent 75 md5: metatags per posts.json query. if (unresolved.length > 0) { try { - for (let index = 0; index < unresolved.length; index += 20) { - const batch = unresolved.slice(index, index + 20); + for ( + let index = 0; + index < unresolved.length; + index += MD5_BATCH_SIZE + ) { + const md5Batch = unresolved.slice(index, index + MD5_BATCH_SIZE); const posts = await fetchPostsByMd5( creds, - batch.map((temp) => temp.md5), + md5Batch.map((temp) => temp.md5), ); for (const post of posts) { - const match = batch.find((temp) => temp.md5 === post.file.md5); + const match = md5Batch.find((temp) => temp.md5 === post.file.md5); if (!match) continue; try { - await api(`/api/uploads/${match.temp_id}/resolve/`, { - method: "POST", - json: { mode: "link", post, auto: true }, - }); + const updated = await api( + `/api/uploads/${match.temp_id}/resolve/`, + { method: "POST", json: { mode: "link", post, auto: true } }, + ); + upsertUpload(updated); } catch { // Leave it pending. } @@ -949,6 +975,23 @@ export default function UploadPage() { void checkSimilarity(unresolvedIqdbIds(fresh)); } + /** Mark an entry done, fade it out and drop it once it has been seen. */ + function finishEntry(entry: UploadEntry, temp: TempUpload) { + updateEntry(entry.id, { status: "done", progress: 100 }); + setBatch((current) => ({ ...current, done: current.done + 1 })); + // The board must move the moment an upload lands, not when the batch ends. + upsertUpload(temp); + window.setTimeout(() => { + updateEntry(entry.id, { fading: true }); + window.setTimeout(() => { + if (entry.previewUrl) URL.revokeObjectURL(entry.previewUrl); + setEntries((current) => + current.filter((item) => item.id !== entry.id), + ); + }, 500); + }, ENTRY_CLEAR_MS); + } + async function drainQueue() { if (runningRef.current) return; runningRef.current = true; @@ -968,13 +1011,14 @@ export default function UploadPage() { const temp = await uploadFile(entry.file, {}, (percent) => updateEntry(entry.id, { progress: percent }), ); - updateEntry(entry.id, { status: "done", progress: 100 }); + finishEntry(entry, temp); created.push(temp); } catch (error) { updateEntry(entry.id, { status: "error", error: errorMessage(error), }); + setBatch((current) => ({ ...current, failed: current.failed + 1 })); } } } @@ -1095,7 +1139,7 @@ export default function UploadPage() { {busy - ? `Uploading ${Math.min(uploadedCount + 1, entries.length)}/${entries.length}…` + ? `Uploading ${Math.min(settledCount + 1, batch.total)}/${batch.total}…` : iqdbProgress ? `Checking IQDB — ${iqdbProgress.done}/${iqdbProgress.total}${ currentCheck ? ` · ${currentCheck}` : "" @@ -1169,8 +1213,8 @@ export default function UploadPage() {

Uploads - {uploadedCount}/{entries.length} uploaded - {failedCount > 0 ? ` · ${failedCount} failed` : ""} + {batch.done}/{batch.total} uploaded + {batch.failed > 0 ? ` · ${batch.failed} failed` : ""}