Match local files to e621: MD5 lookups, manual links, batch scans
- MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621.
This commit is contained in:
@@ -12,7 +12,12 @@ import { isTypingTarget } from "@/lib/dom";
|
||||
import { renderDText } from "@/lib/dtext";
|
||||
import { RATING_LABELS, TAG_CATEGORY_LABELS } from "@/lib/e621";
|
||||
import { formatBytes, formatDate } from "@/lib/format";
|
||||
import type { FileNeighbors, MediaItem, Rating } from "@/lib/types";
|
||||
import type {
|
||||
E621MatchStatus,
|
||||
FileNeighbors,
|
||||
MediaItem,
|
||||
Rating,
|
||||
} from "@/lib/types";
|
||||
import { useAuth } from "@/store/auth";
|
||||
import { useUi } from "@/store/ui";
|
||||
|
||||
@@ -45,6 +50,188 @@ const ratingLabel: Record<string, string> = {
|
||||
e: "Explicit",
|
||||
};
|
||||
|
||||
const matchStatusMeta: Record<
|
||||
E621MatchStatus,
|
||||
{ label: string; className: string }
|
||||
> = {
|
||||
unknown: {
|
||||
label: "Not checked",
|
||||
className: "border-ctp-surface2 bg-ctp-surface0 text-ctp-subtext1",
|
||||
},
|
||||
matched: {
|
||||
label: "Matched",
|
||||
className: "border-ctp-green/40 bg-ctp-green/15 text-ctp-green",
|
||||
},
|
||||
not_found: {
|
||||
label: "Not found on e621",
|
||||
className: "border-ctp-yellow/40 bg-ctp-yellow/15 text-ctp-yellow",
|
||||
},
|
||||
deleted: {
|
||||
label: "Deleted on e621",
|
||||
className: "border-ctp-red/40 bg-ctp-red/15 text-ctp-red",
|
||||
},
|
||||
};
|
||||
|
||||
function MatchCard({ item, canEdit }: { item: MediaItem; canEdit: boolean }) {
|
||||
const queryClient = useQueryClient();
|
||||
const [postId, setPostId] = useState("");
|
||||
const [notice, setNotice] = useState<{
|
||||
kind: "ok" | "warn" | "error";
|
||||
text: string;
|
||||
} | null>(null);
|
||||
|
||||
function refresh() {
|
||||
void queryClient.invalidateQueries({ queryKey: ["file", item.j_id] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["tag-cloud"] });
|
||||
}
|
||||
|
||||
const checkMutation = useMutation({
|
||||
mutationFn: () =>
|
||||
api<MediaItem>(`/api/files/${item.j_id}/match/`, {
|
||||
method: "POST",
|
||||
json: {},
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
refresh();
|
||||
if (data.e621_match_status === "matched") {
|
||||
setNotice({ kind: "ok", text: `Matched to post #${data.e621_post_id}.` });
|
||||
} else if (data.e621_match_status === "deleted") {
|
||||
setNotice({
|
||||
kind: "error",
|
||||
text: "The linked post no longer exists on e621.",
|
||||
});
|
||||
} else {
|
||||
setNotice({
|
||||
kind: "warn",
|
||||
text: "No e621 post with this file's MD5 was found.",
|
||||
});
|
||||
}
|
||||
},
|
||||
onError: (error) =>
|
||||
setNotice({ kind: "error", text: errorMessage(error) }),
|
||||
});
|
||||
|
||||
const linkMutation = useMutation({
|
||||
mutationFn: (id: number) =>
|
||||
api<MediaItem & { md5_match?: boolean }>(`/api/files/${item.j_id}/match/`, {
|
||||
method: "POST",
|
||||
json: { post_id: id },
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
refresh();
|
||||
setPostId("");
|
||||
setNotice(
|
||||
data.md5_match
|
||||
? {
|
||||
kind: "ok",
|
||||
text: `Linked to post #${data.e621_post_id} — MD5 matches.`,
|
||||
}
|
||||
: {
|
||||
kind: "warn",
|
||||
text: `Linked to post #${data.e621_post_id}, but its MD5 differs from this file.`,
|
||||
},
|
||||
);
|
||||
},
|
||||
onError: (error) =>
|
||||
setNotice({ kind: "error", text: errorMessage(error) }),
|
||||
});
|
||||
|
||||
const unlinkMutation = useMutation({
|
||||
mutationFn: () =>
|
||||
api<MediaItem>(`/api/files/${item.j_id}/unlink/`, { method: "POST" }),
|
||||
onSuccess: () => {
|
||||
refresh();
|
||||
setNotice({ kind: "ok", text: "e621 link removed." });
|
||||
},
|
||||
onError: (error) =>
|
||||
setNotice({ kind: "error", text: errorMessage(error) }),
|
||||
});
|
||||
|
||||
const meta = matchStatusMeta[item.e621_match_status];
|
||||
|
||||
return (
|
||||
<section className="rounded-lg border border-ctp-surface0 bg-ctp-base p-4">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<h2 className="text-sm font-semibold text-ctp-subtext1">e621 match</h2>
|
||||
<span
|
||||
className={cn(
|
||||
"rounded-full border px-2 py-0.5 text-[11px] font-medium",
|
||||
meta.className,
|
||||
)}
|
||||
>
|
||||
{meta.label}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<p className="mt-2 text-xs text-ctp-overlay0">
|
||||
{item.e621_checked_at
|
||||
? `Last checked ${formatDate(item.e621_checked_at)}`
|
||||
: "This item has not been checked against e621 yet."}
|
||||
</p>
|
||||
|
||||
{canEdit ? (
|
||||
<div className="mt-3 flex flex-col gap-2">
|
||||
<div className="flex gap-2">
|
||||
<Button
|
||||
variant="secondary"
|
||||
disabled={checkMutation.isPending}
|
||||
onClick={() => checkMutation.mutate()}
|
||||
>
|
||||
{checkMutation.isPending
|
||||
? "Checking…"
|
||||
: item.e621_post_id
|
||||
? "Recheck"
|
||||
: "Check MD5"}
|
||||
</Button>
|
||||
{item.e621_post_id ? (
|
||||
<Button
|
||||
variant="ghost"
|
||||
disabled={unlinkMutation.isPending}
|
||||
onClick={() => unlinkMutation.mutate()}
|
||||
>
|
||||
Unlink
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
className={inputClass}
|
||||
placeholder="Post ID"
|
||||
inputMode="numeric"
|
||||
value={postId}
|
||||
onChange={(event) =>
|
||||
setPostId(event.target.value.replace(/[^0-9]/g, ""))
|
||||
}
|
||||
/>
|
||||
<Button
|
||||
disabled={!postId || linkMutation.isPending}
|
||||
onClick={() => linkMutation.mutate(Number(postId))}
|
||||
>
|
||||
Link
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{notice ? (
|
||||
<p
|
||||
className={cn(
|
||||
"mt-2 text-xs",
|
||||
notice.kind === "ok"
|
||||
? "text-ctp-green"
|
||||
: notice.kind === "warn"
|
||||
? "text-ctp-yellow"
|
||||
: "text-ctp-red",
|
||||
)}
|
||||
>
|
||||
{notice.text}
|
||||
</p>
|
||||
) : null}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function E621MetadataCard({ item }: { item: MediaItem }) {
|
||||
const data = item.e621_data;
|
||||
if (!item.e621_post_id || !data) return null;
|
||||
@@ -77,6 +264,13 @@ function E621MetadataCard({ item }: { item: MediaItem }) {
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{item.e621_match_status === "deleted" ? (
|
||||
<p className="mt-3 rounded border border-ctp-red/40 bg-ctp-red/10 px-2.5 py-1.5 text-xs text-ctp-red">
|
||||
This post has been deleted on e621 — the metadata below is a cached
|
||||
copy.
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<dl className="mt-3 grid grid-cols-2 gap-x-4 gap-y-2 text-xs sm:grid-cols-4">
|
||||
<div>
|
||||
<dt className="text-ctp-overlay0">Rating</dt>
|
||||
@@ -501,6 +695,8 @@ export function LibraryDetail({ jId }: { jId: string }) {
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<MatchCard key={item.j_id} item={item} canEdit={canEdit} />
|
||||
|
||||
<section className="rounded-lg border border-ctp-surface0 bg-ctp-base p-4">
|
||||
<h2 className="text-sm font-semibold text-ctp-subtext1">
|
||||
Locations ({item.locations.length})
|
||||
|
||||
@@ -5,6 +5,7 @@ import { useState, useEffect, type FormEvent } from "react";
|
||||
import { CollapsibleSidebar } from "@/components/CollapsibleSidebar";
|
||||
import { MediaCard } from "@/components/MediaCard";
|
||||
import { Button, EmptyState, Spinner, inputClass } from "@/components/ui";
|
||||
import { MatchPanel } from "@/features/library/MatchPanel";
|
||||
import { api } from "@/lib/api";
|
||||
import { cn } from "@/lib/cn";
|
||||
import { isTypingTarget } from "@/lib/dom";
|
||||
@@ -47,6 +48,16 @@ const statusFilters = [
|
||||
label: "Matched",
|
||||
active: "border-ctp-green/40 bg-ctp-green/15 text-ctp-green",
|
||||
},
|
||||
{
|
||||
value: "not_found",
|
||||
label: "Not found",
|
||||
active: "border-ctp-yellow/40 bg-ctp-yellow/15 text-ctp-yellow",
|
||||
},
|
||||
{
|
||||
value: "deleted",
|
||||
label: "Deleted",
|
||||
active: "border-ctp-red/40 bg-ctp-red/15 text-ctp-red",
|
||||
},
|
||||
{
|
||||
value: "custom",
|
||||
label: "Custom",
|
||||
@@ -323,11 +334,14 @@ export default function LibraryPage() {
|
||||
? `${query.data.count} item${query.data.count === 1 ? "" : "s"}`
|
||||
: "…"}
|
||||
</p>
|
||||
{pageCount > 1 ? (
|
||||
<p className="font-mono text-xs text-ctp-overlay0">
|
||||
Page {page} / {pageCount}
|
||||
</p>
|
||||
) : null}
|
||||
<div className="flex items-center gap-3">
|
||||
{pageCount > 1 ? (
|
||||
<p className="font-mono text-xs text-ctp-overlay0">
|
||||
Page {page} / {pageCount}
|
||||
</p>
|
||||
) : null}
|
||||
{user?.can_upload ? <MatchPanel /> : null}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!user ? (
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { RefreshCw, X } from "lucide-react";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import { Button, Spinner } from "@/components/ui";
|
||||
import { api, errorMessage } from "@/lib/api";
|
||||
import type { MatchTask } from "@/lib/types";
|
||||
|
||||
type Scope = "missing" | "all";
|
||||
|
||||
function summary(task: MatchTask): string {
|
||||
const parts = [
|
||||
`${task.matched} matched`,
|
||||
`${task.not_found} not found`,
|
||||
`${task.deleted} deleted`,
|
||||
];
|
||||
if (task.failed > 0) parts.push(`${task.failed} failed`);
|
||||
return parts.join(" · ");
|
||||
}
|
||||
|
||||
export function MatchPanel() {
|
||||
const queryClient = useQueryClient();
|
||||
const [taskId, setTaskId] = useState<string | null>(null);
|
||||
const [menuOpen, setMenuOpen] = useState(false);
|
||||
|
||||
const taskQuery = useQuery({
|
||||
queryKey: ["match-task", taskId],
|
||||
queryFn: () => api<MatchTask>(`/api/matches/${taskId}/`),
|
||||
enabled: Boolean(taskId),
|
||||
refetchInterval: (query) => {
|
||||
const status = query.state.data?.status;
|
||||
return status === "pending" || status === "running" ? 1000 : false;
|
||||
},
|
||||
});
|
||||
|
||||
const task = taskQuery.data;
|
||||
const running = task?.status === "pending" || task?.status === "running";
|
||||
const finishedTaskId = task && !running ? task.task_id : null;
|
||||
|
||||
useEffect(() => {
|
||||
if (!finishedTaskId) return;
|
||||
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
||||
void queryClient.invalidateQueries({ queryKey: ["tag-cloud"] });
|
||||
}, [finishedTaskId, queryClient]);
|
||||
|
||||
const startMutation = useMutation({
|
||||
mutationFn: (scope: Scope) =>
|
||||
api<MatchTask>("/api/matches/", { method: "POST", json: { scope } }),
|
||||
onSuccess: (data) => {
|
||||
setTaskId(data.task_id);
|
||||
setMenuOpen(false);
|
||||
},
|
||||
});
|
||||
|
||||
const cancelMutation = useMutation({
|
||||
mutationFn: () =>
|
||||
api<{ success: boolean }>(`/api/matches/${taskId}/cancel/`, {
|
||||
method: "POST",
|
||||
}),
|
||||
});
|
||||
|
||||
if (running && task) {
|
||||
const percent = task.total
|
||||
? Math.round((task.processed * 100) / task.total)
|
||||
: 0;
|
||||
return (
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="w-36">
|
||||
<div className="h-1.5 overflow-hidden rounded-full bg-ctp-surface0">
|
||||
<div
|
||||
className="h-full bg-ctp-mauve transition-all"
|
||||
style={{ width: `${percent}%` }}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<span className="font-mono text-xs text-ctp-subtext0">
|
||||
{task.processed}/{task.total} · {task.matched} matched
|
||||
</span>
|
||||
<Button
|
||||
variant="ghost"
|
||||
disabled={cancelMutation.isPending}
|
||||
onClick={() => cancelMutation.mutate()}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (task) {
|
||||
return (
|
||||
<div className="flex items-center gap-2 text-xs">
|
||||
{task.status === "complete" ? (
|
||||
<span className="text-ctp-green">Scan complete — {summary(task)}</span>
|
||||
) : task.status === "cancelled" ? (
|
||||
<span className="text-ctp-subtext0">Scan cancelled.</span>
|
||||
) : (
|
||||
<span className="text-ctp-red">
|
||||
{task.error || "The scan failed."}
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
className="text-ctp-overlay0 transition hover:text-ctp-text"
|
||||
onClick={() => setTaskId(null)}
|
||||
title="Dismiss"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
<Button variant="secondary" onClick={() => setMenuOpen((open) => !open)}>
|
||||
{startMutation.isPending ? (
|
||||
<Spinner className="h-3.5 w-3.5" />
|
||||
) : (
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
)}
|
||||
Match e621
|
||||
</Button>
|
||||
|
||||
{menuOpen ? (
|
||||
<div className="absolute right-0 top-full z-20 mt-2 w-72 rounded-lg border border-ctp-surface1 bg-ctp-base p-1.5 shadow-lg">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full flex-col rounded-md px-2.5 py-2 text-left transition hover:bg-ctp-surface0"
|
||||
onClick={() => startMutation.mutate("missing")}
|
||||
>
|
||||
<span className="text-sm text-ctp-text">
|
||||
Check unchecked items
|
||||
</span>
|
||||
<span className="text-xs text-ctp-overlay0">
|
||||
MD5 lookups for items never checked against e621.
|
||||
</span>
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full flex-col rounded-md px-2.5 py-2 text-left transition hover:bg-ctp-surface0"
|
||||
onClick={() => startMutation.mutate("all")}
|
||||
>
|
||||
<span className="text-sm text-ctp-text">Recheck the library</span>
|
||||
<span className="text-xs text-ctp-overlay0">
|
||||
Refreshes matched metadata and flags deleted posts too.
|
||||
</span>
|
||||
</button>
|
||||
{startMutation.isError ? (
|
||||
<p className="px-2.5 py-1.5 text-xs text-ctp-red">
|
||||
{errorMessage(startMutation.error)}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -81,6 +81,8 @@ export interface MediaLocation {
|
||||
|
||||
export type Rating = "" | "s" | "q" | "e";
|
||||
|
||||
export type E621MatchStatus = "unknown" | "matched" | "not_found" | "deleted";
|
||||
|
||||
export interface E621StoredPost {
|
||||
id: number | null;
|
||||
created_at?: string | null;
|
||||
@@ -126,6 +128,8 @@ export interface MediaItem {
|
||||
uploaded_by_id: number | null;
|
||||
e621_post_id: number | null;
|
||||
e621_data: E621StoredPost | null;
|
||||
e621_match_status: E621MatchStatus;
|
||||
e621_checked_at: string | null;
|
||||
locations: MediaLocation[];
|
||||
raw_url: string;
|
||||
thumbnail_url: string;
|
||||
@@ -185,6 +189,21 @@ export interface DownloadTask {
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface MatchTask {
|
||||
task_id: string;
|
||||
scope: "missing" | "all";
|
||||
status: "pending" | "running" | "complete" | "error" | "cancelled";
|
||||
total: number;
|
||||
processed: number;
|
||||
matched: number;
|
||||
not_found: number;
|
||||
deleted: number;
|
||||
failed: number;
|
||||
error: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface E621IqdbCandidate {
|
||||
post_id: number | null;
|
||||
score: number | null;
|
||||
|
||||
Reference in New Issue
Block a user