Live-updating upload board, self-clearing tiles, original MD5 batch size

- MD5 auto-match now sends 75 md5: metatags per posts.json query, the same
  batch size the original J621-Django app used (was 20); the limit cap no
  longer truncates batches.
- every settled upload is upserted into the staged list right away, so the
  Pending / Visual Similarity / Auto-uploaded columns move as files land
  instead of waiting for the whole batch (auto-matched resolves and IQDB
  results use the same path).
- finished upload tiles fade out and remove themselves ~2s after completing;
  failures stay until cleared. Batch counters are tracked separately from the
  visible tiles so the header and progress bar stay accurate as tiles vanish.
- AGENTS.md now points at the original Django app for reference behavior.

Verified live with a headless upload run: the columns showed the new files
immediately and the 8 tiles were gone ~3s after finishing.
This commit is contained in:
2026-09-19 01:18:50 -05:00
parent 9641862515
commit e62d7af42f
3 changed files with 88 additions and 36 deletions
+82 -35
View File
@@ -41,12 +41,19 @@ type UploadStatus = "queued" | "uploading" | "done" | "error";
/** How many files upload at once; big batches were painfully serial. */
const UPLOAD_CONCURRENCY = 3;
/** md5: metatags per posts.json query — same batch size as the original app. */
const MD5_BATCH_SIZE = 75;
/** Finished upload tiles clear themselves this long after completing. */
const ENTRY_CLEAR_MS = 2000;
interface UploadEntry {
id: string;
file: File;
progress: number;
status: UploadStatus;
previewUrl: string;
fading?: boolean;
error?: string;
}
@@ -700,6 +707,7 @@ export default function UploadPage() {
const objectUrls = useRef<string[]>([]);
const [dragging, setDragging] = useState(false);
const [entries, setEntries] = useState<UploadEntry[]>([]);
const [batch, setBatch] = useState({ total: 0, done: 0, failed: 0 });
const [busy, setBusy] = useState(false);
const [processing, setProcessing] = useState(false);
const [checkingIds, setCheckingIds] = useState<string[]>([]);
@@ -746,20 +754,18 @@ export default function UploadPage() {
(entry) => entry.status === "uploading",
);
const uploadedCount = entries.filter((entry) => entry.status === "done").length;
const failedCount = entries.filter((entry) => entry.status === "error").length;
const batchPercent = entries.length
? Math.round(
entries.reduce(
(total, entry) =>
total +
(entry.status === "done"
? 100
: entry.status === "queued"
? 0
: entry.progress),
0,
) / entries.length,
// Finished tiles clear themselves after a moment, so the counters come
// from batch totals rather than the tiles currently on screen.
const settledCount = batch.done + batch.failed;
const inflightProgress = entries.reduce(
(total, entry) =>
total + (entry.status === "uploading" ? entry.progress : 0),
0,
);
const batchPercent = batch.total
? Math.min(
100,
Math.round((settledCount * 100 + inflightProgress) / batch.total),
)
: 0;
@@ -780,6 +786,15 @@ export default function UploadPage() {
};
});
setEntries((current) => [...current, ...next]);
// Start a fresh batch when the previous one is fully settled.
setBatch((current) => {
const settled = current.done + current.failed >= current.total;
return {
total: (settled ? 0 : current.total) + next.length,
done: settled ? 0 : current.done,
failed: settled ? 0 : current.failed,
};
});
queueRef.current.push(...next);
void drainQueue();
}
@@ -800,6 +815,17 @@ export default function UploadPage() {
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
}
/** Patch one staged upload into the list so the board moves as work happens. */
function upsertUpload(temp: TempUpload) {
queryClient.setQueryData<TempUpload[]>(["uploads"], (current) => {
if (!current) return current;
// The API lists newest first, so unseen uploads go on top.
return current.some((item) => item.temp_id === temp.temp_id)
? current.map((item) => (item.temp_id === temp.temp_id ? temp : item))
: [temp, ...current];
});
}
async function refreshUploads(): Promise<TempUpload[]> {
const fresh = await api<TempUpload[]>("/api/uploads/");
queryClient.setQueryData(["uploads"], fresh);
@@ -870,13 +896,7 @@ export default function UploadPage() {
`/api/uploads/${temp.temp_id}/iqdb/`,
{ method: "POST", json: { results } },
);
queryClient.setQueryData<TempUpload[]>(["uploads"], (current) =>
current
? current.map((item) =>
item.temp_id === updated.temp_id ? updated : item,
)
: current,
);
upsertUpload(updated);
} catch (error) {
setCheckErrors((current) => ({
...current,
@@ -915,23 +935,29 @@ export default function UploadPage() {
const creds = effectiveCredentials(credentials);
const unresolved = created.filter((temp) => temp.status === "pending");
// 1) Auto-complete files whose MD5 matches an e621 post.
// 1) Auto-complete files whose MD5 matches an e621 post. The original app
// sent 75 md5: metatags per posts.json query.
if (unresolved.length > 0) {
try {
for (let index = 0; index < unresolved.length; index += 20) {
const batch = unresolved.slice(index, index + 20);
for (
let index = 0;
index < unresolved.length;
index += MD5_BATCH_SIZE
) {
const md5Batch = unresolved.slice(index, index + MD5_BATCH_SIZE);
const posts = await fetchPostsByMd5(
creds,
batch.map((temp) => temp.md5),
md5Batch.map((temp) => temp.md5),
);
for (const post of posts) {
const match = batch.find((temp) => temp.md5 === post.file.md5);
const match = md5Batch.find((temp) => temp.md5 === post.file.md5);
if (!match) continue;
try {
await api(`/api/uploads/${match.temp_id}/resolve/`, {
method: "POST",
json: { mode: "link", post, auto: true },
});
const updated = await api<TempUpload>(
`/api/uploads/${match.temp_id}/resolve/`,
{ method: "POST", json: { mode: "link", post, auto: true } },
);
upsertUpload(updated);
} catch {
// Leave it pending.
}
@@ -949,6 +975,23 @@ export default function UploadPage() {
void checkSimilarity(unresolvedIqdbIds(fresh));
}
/** Mark an entry done, fade it out and drop it once it has been seen. */
function finishEntry(entry: UploadEntry, temp: TempUpload) {
updateEntry(entry.id, { status: "done", progress: 100 });
setBatch((current) => ({ ...current, done: current.done + 1 }));
// The board must move the moment an upload lands, not when the batch ends.
upsertUpload(temp);
window.setTimeout(() => {
updateEntry(entry.id, { fading: true });
window.setTimeout(() => {
if (entry.previewUrl) URL.revokeObjectURL(entry.previewUrl);
setEntries((current) =>
current.filter((item) => item.id !== entry.id),
);
}, 500);
}, ENTRY_CLEAR_MS);
}
async function drainQueue() {
if (runningRef.current) return;
runningRef.current = true;
@@ -968,13 +1011,14 @@ export default function UploadPage() {
const temp = await uploadFile(entry.file, {}, (percent) =>
updateEntry(entry.id, { progress: percent }),
);
updateEntry(entry.id, { status: "done", progress: 100 });
finishEntry(entry, temp);
created.push(temp);
} catch (error) {
updateEntry(entry.id, {
status: "error",
error: errorMessage(error),
});
setBatch((current) => ({ ...current, failed: current.failed + 1 }));
}
}
}
@@ -1095,7 +1139,7 @@ export default function UploadPage() {
<Spinner className="h-3.5 w-3.5 shrink-0" />
<span className="truncate">
{busy
? `Uploading ${Math.min(uploadedCount + 1, entries.length)}/${entries.length}…`
? `Uploading ${Math.min(settledCount + 1, batch.total)}/${batch.total}…`
: iqdbProgress
? `Checking IQDB — ${iqdbProgress.done}/${iqdbProgress.total}${
currentCheck ? ` · ${currentCheck}` : ""
@@ -1169,8 +1213,8 @@ export default function UploadPage() {
<h2 className="text-sm font-semibold text-ctp-subtext1">
Uploads
<span className="ml-2 font-mono text-[11px] text-ctp-overlay0">
{uploadedCount}/{entries.length} uploaded
{failedCount > 0 ? ` · ${failedCount} failed` : ""}
{batch.done}/{batch.total} uploaded
{batch.failed > 0 ? ` · ${batch.failed} failed` : ""}
</span>
</h2>
<button
@@ -1225,7 +1269,10 @@ export default function UploadPage() {
{entries.map((entry) => (
<div
key={entry.id}
className="overflow-hidden rounded-lg border border-ctp-surface0 bg-ctp-base"
className={cn(
"overflow-hidden rounded-lg border border-ctp-surface0 bg-ctp-base transition-opacity duration-500",
entry.fading && "opacity-0",
)}
>
<div className="relative aspect-square overflow-hidden bg-ctp-mantle">
{entry.previewUrl ? (