Make upload processing phase-based: MD5 -> visual -> IQDB over the batch

Uploads were doing md5 + local visual matching inside the upload request
(backend create) while the frontend later ran its own e621 MD5 pass, so the
pipeline looked interleaved per file. Now every step is a phase applied to
the whole batch in order:

1. upload (fast: md5 + exact-duplicate check only),
2. e621 MD5 lookup, 75 md5: metatags per posts.json request,
3. local visual similarity, one file at a time via the new
   POST /api/uploads/<id>/visual-match/ action,
4. IQDB through the existing serial queue.

The board shows the active phase with its own progress bar (e621 MD5 in
peach, visual in lavender, IQDB in teal) and every step updates the staged
list as it lands. Verified from a headless run: one batched posts.json
request for 10 files, then 10 visual-match calls, then IQDB.

frontend/src/features/upload/UploadPage.tsx

@@ -127,6 +127,32 @@ function isFatalIqdbError(error: unknown): boolean {
   return error instanceof TypeError || error instanceof DOMException;
 }
 
+/** Small labeled progress bar for one upload-pipeline phase. */
+function PhaseBar({
+  label,
+  percent,
+  tone,
+}: {
+  label: string;
+  percent: number;
+  tone: string;
+}) {
+  return (
+    <div className="flex w-64 flex-col gap-1.5">
+      <p className="flex items-center gap-2 text-xs text-ctp-overlay0">
+        <Spinner className="h-3.5 w-3.5 shrink-0" />
+        <span className="truncate">{label}</span>
+      </p>
+      <div className="h-1 overflow-hidden rounded-full bg-ctp-surface0">
+        <div
+          className={cn("h-full transition-[width]", tone)}
+          style={{ width: `${Math.min(100, Math.max(0, percent))}%` }}
+        />
+      </div>
+    </div>
+  );
+}
+
 function TempCard({
   temp,
   checking,
@@ -708,6 +734,11 @@ export default function UploadPage() {
   const [dragging, setDragging] = useState(false);
   const [entries, setEntries] = useState<UploadEntry[]>([]);
   const [batch, setBatch] = useState({ total: 0, done: 0, failed: 0 });
+  const [pipeline, setPipeline] = useState<{
+    phase: "md5" | "visual";
+    done: number;
+    total: number;
+  } | null>(null);
   const [busy, setBusy] = useState(false);
   const [processing, setProcessing] = useState(false);
   const [checkingIds, setCheckingIds] = useState<string[]>([]);
@@ -931,48 +962,83 @@ export default function UploadPage() {
     }
   }
 
-  async function autoMatchAndCheck(created: TempUpload[]) {
+  /**
+   * Phased processing for one finished upload batch: every file goes through
+   * the e621 MD5 lookup (batched), then local visual similarity, then IQDB.
+   * Phases never interleave per file, and each step lands on the board as it
+   * finishes.
+   */
+  async function runUploadPipeline(created: TempUpload[]) {
     const creds = effectiveCredentials(credentials);
-    const unresolved = created.filter((temp) => temp.status === "pending");
+    const targets = created.filter((temp) => temp.status === "pending");
+    if (targets.length === 0) {
+      invalidateUploads();
+      return;
+    }
+    const targetIds = new Set(targets.map((temp) => temp.temp_id));
 
-    // 1) Auto-complete files whose MD5 matches an e621 post. The original app
-    // sent 75 md5: metatags per posts.json query.
-    if (unresolved.length > 0) {
-      try {
-        for (
-          let index = 0;
-          index < unresolved.length;
-          index += MD5_BATCH_SIZE
-        ) {
-          const md5Batch = unresolved.slice(index, index + MD5_BATCH_SIZE);
-          const posts = await fetchPostsByMd5(
-            creds,
-            md5Batch.map((temp) => temp.md5),
-          );
-          for (const post of posts) {
-            const match = md5Batch.find((temp) => temp.md5 === post.file.md5);
-            if (!match) continue;
-            try {
-              const updated = await api<TempUpload>(
-                `/api/uploads/${match.temp_id}/resolve/`,
-                { method: "POST", json: { mode: "link", post, auto: true } },
-              );
-              upsertUpload(updated);
-            } catch {
-              // Leave it pending.
-            }
+    // Phase 1: e621 MD5 lookup, 75 md5: metatags per query.
+    setPipeline({ phase: "md5", done: 0, total: targets.length });
+    try {
+      for (let index = 0; index < targets.length; index += MD5_BATCH_SIZE) {
+        const md5Batch = targets.slice(index, index + MD5_BATCH_SIZE);
+        const posts = await fetchPostsByMd5(
+          creds,
+          md5Batch.map((temp) => temp.md5),
+        );
+        for (const post of posts) {
+          const match = md5Batch.find((temp) => temp.md5 === post.file.md5);
+          if (!match) continue;
+          try {
+            const updated = await api<TempUpload>(
+              `/api/uploads/${match.temp_id}/resolve/`,
+              { method: "POST", json: { mode: "link", post, auto: true } },
+            );
+            upsertUpload(updated);
+          } catch {
+            // Leave it pending.
           }
         }
-      } catch {
-        // e621 unavailable; leave everything pending.
+        setPipeline((current) =>
+          current
+            ? {
+                ...current,
+                done: Math.min(index + MD5_BATCH_SIZE, targets.length),
+              }
+            : current,
+        );
       }
-      invalidateUploads();
+    } catch {
+      // e621 unavailable; those files stay pending.
     }
+    setPipeline(null);
 
-    // 2) IQDB similarity for every unresolved file that has not been checked
-    // yet (pending and local visual matches alike; IQDB is image-only).
+    // Phase 2: local visual similarity, one file at a time.
+    const cached = queryClient.getQueryData<TempUpload[]>(["uploads"]) ?? [];
+    const visualTargets = cached.filter(
+      (temp) => targetIds.has(temp.temp_id) && temp.status === "pending",
+    );
+    setPipeline({ phase: "visual", done: 0, total: visualTargets.length });
+    for (const temp of visualTargets) {
+      try {
+        const updated = await api<TempUpload>(
+          `/api/uploads/${temp.temp_id}/visual-match/`,
+          { method: "POST" },
+        );
+        upsertUpload(updated);
+      } catch {
+        // Keep it pending.
+      }
+      setPipeline((current) =>
+        current ? { ...current, done: current.done + 1 } : current,
+      );
+    }
+    setPipeline(null);
+
+    // Phase 3: IQDB for everything still unresolved (serial queue with its
+    // own progress, cooldown and fatal-error stop).
     const fresh = await refreshUploads();
-    void checkSimilarity(unresolvedIqdbIds(fresh));
+    checkSimilarity(unresolvedIqdbIds(fresh));
   }
 
   /** Mark an entry done, fade it out and drop it once it has been seen. */
@@ -1032,7 +1098,7 @@ export default function UploadPage() {
     invalidateUploads();
     if (created.length > 0) {
       setProcessing(true);
-      void autoMatchAndCheck(created).finally(() => setProcessing(false));
+      void runUploadPipeline(created).finally(() => setProcessing(false));
     }
   }
 
@@ -1133,33 +1199,38 @@ export default function UploadPage() {
               retry checks
             </button>
           </div>
-        ) : busy || processing ? (
-          <div className="flex w-64 flex-col gap-1.5">
-            <p className="flex items-center gap-2 text-xs text-ctp-overlay0">
-              <Spinner className="h-3.5 w-3.5 shrink-0" />
-              <span className="truncate">
-                {busy
-                  ? `Uploading ${Math.min(settledCount + 1, batch.total)}/${batch.total}…`
-                  : iqdbProgress
-                    ? `Checking IQDB — ${iqdbProgress.done}/${iqdbProgress.total}${
-                        currentCheck ? ` · ${currentCheck}` : ""
-                      }`
-                    : "Processing uploads…"}
-              </span>
-            </p>
-            {!busy && iqdbProgress ? (
-              <div className="h-1 overflow-hidden rounded-full bg-ctp-surface0">
-                <div
-                  className="h-full bg-ctp-teal transition-[width]"
-                  style={{
-                    width: `${Math.round(
-                      (iqdbProgress.done / iqdbProgress.total) * 100,
-                    )}%`,
-                  }}
-                />
-              </div>
-            ) : null}
-          </div>
+        ) : busy ? (
+          <PhaseBar
+            label={`Uploading ${Math.min(settledCount + 1, batch.total)}/${batch.total}…`}
+            percent={batchPercent}
+            tone="bg-ctp-mauve"
+          />
+        ) : pipeline ? (
+          <PhaseBar
+            label={`${
+              pipeline.phase === "md5" ? "e621 MD5" : "Visual similarity"
+            } — ${pipeline.done}/${pipeline.total}`}
+            percent={
+              pipeline.total
+                ? Math.round((pipeline.done / pipeline.total) * 100)
+                : 0
+            }
+            tone={pipeline.phase === "md5" ? "bg-ctp-peach" : "bg-ctp-lavender"}
+          />
+        ) : processing && iqdbProgress ? (
+          <PhaseBar
+            label={`Checking IQDB — ${iqdbProgress.done}/${iqdbProgress.total}${
+              currentCheck ? ` · ${currentCheck}` : ""
+            }`}
+            percent={Math.round(
+              (iqdbProgress.done / iqdbProgress.total) * 100,
+            )}
+            tone="bg-ctp-teal"
+          />
+        ) : processing ? (
+          <p className="flex items-center gap-2 text-xs text-ctp-overlay0">
+            <Spinner className="h-3.5 w-3.5" /> Processing uploads…
+          </p>
         ) : null}
       </header>