Live-updating upload board, self-clearing tiles, original MD5 batch size

- MD5 auto-match now sends 75 md5: metatags per posts.json query, the same
  batch size the original J621-Django app used (was 20); the limit cap no
  longer truncates batches.
- every settled upload is upserted into the staged list right away, so the
  Pending / Visual Similarity / Auto-uploaded columns move as files land
  instead of waiting for the whole batch (auto-matched resolves and IQDB
  results use the same path).
- finished upload tiles fade out and remove themselves ~2s after completing;
  failures stay until cleared. Batch counters are tracked separately from the
  visible tiles so the header and progress bar stay accurate as tiles vanish.
- AGENTS.md now points at the original Django app for reference behavior.

Verified live with a headless upload run: the columns showed the new files
immediately and the 8 tiles were gone ~3s after finishing.

AGENTS.md

@@ -2,6 +2,11 @@ Use the venv on backend/venv/
 
 DO NOT mess with the system's python
 
+The original J621 (Django) app is kept for reference at
+    /mnt/Disco/Proyects/Python3.13/J621-Django/
+Consult it when matching original behavior (batch sizes for e621 lookups,
+per-page sets, upload flow) instead of guessing.
+
 For Frontend work:
 
 read both Frontend Design related Markdown files on the frontend/ folder

frontend/src/features/upload/UploadPage.tsx

@@ -41,12 +41,19 @@ type UploadStatus = "queued" | "uploading" | "done" | "error";
 /** How many files upload at once; big batches were painfully serial. */
 const UPLOAD_CONCURRENCY = 3;
 
+/** md5: metatags per posts.json query — same batch size as the original app. */
+const MD5_BATCH_SIZE = 75;
+
+/** Finished upload tiles clear themselves this long after completing. */
+const ENTRY_CLEAR_MS = 2000;
+
 interface UploadEntry {
   id: string;
   file: File;
   progress: number;
   status: UploadStatus;
   previewUrl: string;
+  fading?: boolean;
   error?: string;
 }
 
@@ -700,6 +707,7 @@ export default function UploadPage() {
   const objectUrls = useRef<string[]>([]);
   const [dragging, setDragging] = useState(false);
   const [entries, setEntries] = useState<UploadEntry[]>([]);
+  const [batch, setBatch] = useState({ total: 0, done: 0, failed: 0 });
   const [busy, setBusy] = useState(false);
   const [processing, setProcessing] = useState(false);
   const [checkingIds, setCheckingIds] = useState<string[]>([]);
@@ -746,20 +754,18 @@ export default function UploadPage() {
     (entry) => entry.status === "uploading",
   );
 
-  const uploadedCount = entries.filter((entry) => entry.status === "done").length;
+  // Finished tiles clear themselves after a moment, so the counters come
-  const failedCount = entries.filter((entry) => entry.status === "error").length;
+  // from batch totals rather than the tiles currently on screen.
-  const batchPercent = entries.length
+  const settledCount = batch.done + batch.failed;
-    ? Math.round(
+  const inflightProgress = entries.reduce(
-        entries.reduce(
     (total, entry) =>
-            total +
+      total + (entry.status === "uploading" ? entry.progress : 0),
-            (entry.status === "done"
-              ? 100
-              : entry.status === "queued"
-                ? 0
-                : entry.progress),
     0,
-        ) / entries.length,
+  );
+  const batchPercent = batch.total
+    ? Math.min(
+        100,
+        Math.round((settledCount * 100 + inflightProgress) / batch.total),
       )
     : 0;
 
@@ -780,6 +786,15 @@ export default function UploadPage() {
       };
     });
     setEntries((current) => [...current, ...next]);
+    // Start a fresh batch when the previous one is fully settled.
+    setBatch((current) => {
+      const settled = current.done + current.failed >= current.total;
+      return {
+        total: (settled ? 0 : current.total) + next.length,
+        done: settled ? 0 : current.done,
+        failed: settled ? 0 : current.failed,
+      };
+    });
     queueRef.current.push(...next);
     void drainQueue();
   }
@@ -800,6 +815,17 @@ export default function UploadPage() {
     void queryClient.invalidateQueries({ queryKey: ["uploads"] });
   }
 
+  /** Patch one staged upload into the list so the board moves as work happens. */
+  function upsertUpload(temp: TempUpload) {
+    queryClient.setQueryData<TempUpload[]>(["uploads"], (current) => {
+      if (!current) return current;
+      // The API lists newest first, so unseen uploads go on top.
+      return current.some((item) => item.temp_id === temp.temp_id)
+        ? current.map((item) => (item.temp_id === temp.temp_id ? temp : item))
+        : [temp, ...current];
+    });
+  }
+
   async function refreshUploads(): Promise<TempUpload[]> {
     const fresh = await api<TempUpload[]>("/api/uploads/");
     queryClient.setQueryData(["uploads"], fresh);
@@ -870,13 +896,7 @@ export default function UploadPage() {
             `/api/uploads/${temp.temp_id}/iqdb/`,
             { method: "POST", json: { results } },
           );
-          queryClient.setQueryData<TempUpload[]>(["uploads"], (current) =>
+          upsertUpload(updated);
-            current
-              ? current.map((item) =>
-                  item.temp_id === updated.temp_id ? updated : item,
-                )
-              : current,
-          );
         } catch (error) {
           setCheckErrors((current) => ({
             ...current,
@@ -915,23 +935,29 @@ export default function UploadPage() {
     const creds = effectiveCredentials(credentials);
     const unresolved = created.filter((temp) => temp.status === "pending");
 
-    // 1) Auto-complete files whose MD5 matches an e621 post.
+    // 1) Auto-complete files whose MD5 matches an e621 post. The original app
+    // sent 75 md5: metatags per posts.json query.
     if (unresolved.length > 0) {
       try {
-        for (let index = 0; index < unresolved.length; index += 20) {
+        for (
-          const batch = unresolved.slice(index, index + 20);
+          let index = 0;
+          index < unresolved.length;
+          index += MD5_BATCH_SIZE
+        ) {
+          const md5Batch = unresolved.slice(index, index + MD5_BATCH_SIZE);
           const posts = await fetchPostsByMd5(
             creds,
-            batch.map((temp) => temp.md5),
+            md5Batch.map((temp) => temp.md5),
           );
           for (const post of posts) {
-            const match = batch.find((temp) => temp.md5 === post.file.md5);
+            const match = md5Batch.find((temp) => temp.md5 === post.file.md5);
             if (!match) continue;
             try {
-              await api(`/api/uploads/${match.temp_id}/resolve/`, {
+              const updated = await api<TempUpload>(
-                method: "POST",
+                `/api/uploads/${match.temp_id}/resolve/`,
-                json: { mode: "link", post, auto: true },
+                { method: "POST", json: { mode: "link", post, auto: true } },
-              });
+              );
+              upsertUpload(updated);
             } catch {
               // Leave it pending.
             }
@@ -949,6 +975,23 @@ export default function UploadPage() {
     void checkSimilarity(unresolvedIqdbIds(fresh));
   }
 
+  /** Mark an entry done, fade it out and drop it once it has been seen. */
+  function finishEntry(entry: UploadEntry, temp: TempUpload) {
+    updateEntry(entry.id, { status: "done", progress: 100 });
+    setBatch((current) => ({ ...current, done: current.done + 1 }));
+    // The board must move the moment an upload lands, not when the batch ends.
+    upsertUpload(temp);
+    window.setTimeout(() => {
+      updateEntry(entry.id, { fading: true });
+      window.setTimeout(() => {
+        if (entry.previewUrl) URL.revokeObjectURL(entry.previewUrl);
+        setEntries((current) =>
+          current.filter((item) => item.id !== entry.id),
+        );
+      }, 500);
+    }, ENTRY_CLEAR_MS);
+  }
+
   async function drainQueue() {
     if (runningRef.current) return;
     runningRef.current = true;
@@ -968,13 +1011,14 @@ export default function UploadPage() {
           const temp = await uploadFile(entry.file, {}, (percent) =>
             updateEntry(entry.id, { progress: percent }),
           );
-          updateEntry(entry.id, { status: "done", progress: 100 });
+          finishEntry(entry, temp);
           created.push(temp);
         } catch (error) {
           updateEntry(entry.id, {
             status: "error",
             error: errorMessage(error),
           });
+          setBatch((current) => ({ ...current, failed: current.failed + 1 }));
         }
       }
     }
@@ -1095,7 +1139,7 @@ export default function UploadPage() {
               <Spinner className="h-3.5 w-3.5 shrink-0" />
               <span className="truncate">
                 {busy
-                  ? `Uploading ${Math.min(uploadedCount + 1, entries.length)}/${entries.length}…`
+                  ? `Uploading ${Math.min(settledCount + 1, batch.total)}/${batch.total}…`
                   : iqdbProgress
                     ? `Checking IQDB — ${iqdbProgress.done}/${iqdbProgress.total}${
                         currentCheck ? ` · ${currentCheck}` : ""
@@ -1169,8 +1213,8 @@ export default function UploadPage() {
             <h2 className="text-sm font-semibold text-ctp-subtext1">
               Uploads
               <span className="ml-2 font-mono text-[11px] text-ctp-overlay0">
-                {uploadedCount}/{entries.length} uploaded
+                {batch.done}/{batch.total} uploaded
-                {failedCount > 0 ? ` · ${failedCount} failed` : ""}
+                {batch.failed > 0 ? ` · ${batch.failed} failed` : ""}
               </span>
             </h2>
             <button
@@ -1225,7 +1269,10 @@ export default function UploadPage() {
             {entries.map((entry) => (
               <div
                 key={entry.id}
-                className="overflow-hidden rounded-lg border border-ctp-surface0 bg-ctp-base"
+                className={cn(
+                  "overflow-hidden rounded-lg border border-ctp-surface0 bg-ctp-base transition-opacity duration-500",
+                  entry.fading && "opacity-0",
+                )}
               >
                 <div className="relative aspect-square overflow-hidden bg-ctp-mantle">
                   {entry.previewUrl ? (

frontend/src/lib/e621.ts

@@ -373,7 +373,7 @@ export async function fetchPostsByMd5(
     {
       params: {
         tags: `md5:${md5s.join(",")}`,
-        limit: Math.min(md5s.length, 100),
+        limit: Math.min(md5s.length, 320),
       },
       signal,
     },