Upload pipeline: staging, MD5 auto-match, IQDB, three-column board

Backend:
- TempUpload model: staged files (pending / visual_match / completed /
  error) with resolution, e621 payload, custom metadata and IQDB data
- Files land in a temp folder and only move into the watched library
  folder once resolved; duplicates resolve immediately without a copy
- Endpoints: stage (multipart), list, retrieve, temp file, IQDB save,
  resolve (link to post or custom metadata), discard/dismiss
- cleanup_temp_uploads command for old staged files
- Replaces the old direct-to-library upload endpoint

Frontend:
- Upload page is now a three-column board (Pending & Unmatched /
  Visual Similarity Detected / Auto-uploaded & Indexed)
- After upload: MD5s are batch-checked against e621 and matches
  auto-complete with full post metadata; remaining files run through
  IQDB and move to the similarity column when candidates exist
- Metadata modal with IQDB candidates, post-ID linking and custom
  tags/rating/notes; discard and dismiss actions
- e621 client gains fetchPostsByMd5 and iqdbSearch helpers

Roadmap updated with the completed upload items.
This commit is contained in:
2026-09-17 11:12:03 -05:00
parent 60e1113231
commit d0e2901c92
13 changed files with 1189 additions and 118 deletions
+10 -3
View File
@@ -1,4 +1,4 @@
import type { MediaItem } from "@/lib/types";
import type { TempUpload } from "@/lib/types";
const TOKEN_KEY = "j621.token";
@@ -6,6 +6,13 @@ export function getToken(): string | null {
return window.localStorage.getItem(TOKEN_KEY);
}
export function authHeaders(extra?: HeadersInit): Headers {
const headers = new Headers(extra);
const token = getToken();
if (token) headers.set("Authorization", `Token ${token}`);
return headers;
}
export function setToken(token: string | null): void {
if (token) {
window.localStorage.setItem(TOKEN_KEY, token);
@@ -82,7 +89,7 @@ export function uploadFile(
file: File,
fields: Record<string, string> = {},
onProgress?: (percent: number) => void,
): Promise<MediaItem> {
): Promise<TempUpload> {
return new Promise((resolve, reject) => {
const form = new FormData();
form.append("file", file);
@@ -107,7 +114,7 @@ export function uploadFile(
data = null;
}
if (request.status >= 200 && request.status < 300) {
resolve(data as MediaItem);
resolve(data as TempUpload);
} else {
reject(new ApiError(request.status, data));
}
+84
View File
@@ -296,6 +296,90 @@ export async function setFavorite(
});
}
export async function fetchPostsByMd5(
credentials: E621Credentials,
md5s: string[],
signal?: AbortSignal,
): Promise<E621Post[]> {
if (md5s.length === 0) return [];
const data = await e621Request<{ posts: E621Post[] }>(
credentials,
"/posts.json",
{
params: {
tags: `md5:${md5s.join(",")}`,
limit: Math.min(md5s.length, 100),
},
signal,
},
);
return data.posts;
}
export interface E621IqdbResult {
post_id: number | null;
score: number | null;
preview_url: string | null;
rating: string | null;
md5: string | null;
}
function normalizeIqdbEntry(entry: unknown): E621IqdbResult | null {
if (!entry || typeof entry !== "object") return null;
const record = entry as Record<string, unknown>;
const postWrap = record.post;
let post: Record<string, unknown> = {};
if (postWrap && typeof postWrap === "object") {
const wrap = postWrap as Record<string, unknown>;
post =
wrap.posts && typeof wrap.posts === "object"
? (wrap.posts as Record<string, unknown>)
: wrap;
}
const file = (post.file ?? {}) as Record<string, unknown>;
const preview = (post.preview ?? {}) as Record<string, unknown>;
const postId =
typeof record.post_id === "number"
? record.post_id
: typeof post.id === "number"
? post.id
: null;
return {
post_id: postId,
score: typeof record.score === "number" ? record.score : null,
preview_url: typeof preview.url === "string" ? preview.url : null,
rating: typeof post.rating === "string" ? post.rating : null,
md5: typeof file.md5 === "string" ? file.md5 : null,
};
}
export async function iqdbSearch(
credentials: E621Credentials,
file: Blob,
filename: string,
signal?: AbortSignal,
): Promise<E621IqdbResult[]> {
const form = new FormData();
form.append("file", file, filename);
const data = await e621Request<unknown>(credentials, "/iqdb_queries.json", {
method: "POST",
body: form,
signal,
});
const list = Array.isArray(data)
? data
: Array.isArray((data as { matches?: unknown[] })?.matches)
? (data as { matches: unknown[] }).matches
: [];
return list
.map(normalizeIqdbEntry)
.filter(
(entry): entry is E621IqdbResult =>
entry !== null && (entry.post_id !== null || entry.preview_url !== null),
)
.slice(0, 10);
}
export function postIsBlacklisted(
post: E621Post,
blacklist: Set<string>,
+27
View File
@@ -106,3 +106,30 @@ export interface FileNeighbors {
previous: FileNeighbor | null;
next: FileNeighbor | null;
}
export interface E621IqdbCandidate {
post_id: number | null;
score: number | null;
preview_url: string | null;
rating: string | null;
md5: string | null;
}
export interface TempUpload {
temp_id: string;
original_filename: string;
md5: string;
size: number;
status: "pending" | "visual_match" | "completed" | "error";
resolution: "" | "auto_md5" | "duplicate" | "linked" | "custom";
e621_post_id: number | null;
e621_data: E621StoredPost | null;
custom_rating: Rating;
custom_tags: string[];
custom_notes: string;
iqdb_data: E621IqdbCandidate[] | null;
library_j_id: string | null;
file_url: string | null;
created_at: string;
updated_at: string;
}