Enrich IQDB candidates; link downloads the e621 original
- IQDB responses carry no preview/file data, so candidates only showed an ID; the SPA now enriches them with one batched posts lookup (preview, rating, score, favourites, dimensions, tag preview) - Candidate tiles are selectable instead of instantly resolving: picking one shows its info and an explicit 'Link selected post' button - Linking a post now fetches the e621 original into the library and drops the staged upload; when the staged file's MD5 already equals the post's file, the staged copy is moved instead (identical bytes) - Keep the file URL in stored e621 metadata; sanitize the new candidate fields server-side
This commit is contained in:
@@ -231,6 +231,7 @@ def trim_e621_post(post):
|
|||||||
"size": file_data.get("size"),
|
"size": file_data.get("size"),
|
||||||
"width": file_data.get("width"),
|
"width": file_data.get("width"),
|
||||||
"height": file_data.get("height"),
|
"height": file_data.get("height"),
|
||||||
|
"url": file_data.get("url"),
|
||||||
},
|
},
|
||||||
"uploader_name": post.get("uploader_name"),
|
"uploader_name": post.get("uploader_name"),
|
||||||
}
|
}
|
||||||
@@ -244,12 +245,38 @@ def sanitize_iqdb_results(results):
|
|||||||
continue
|
continue
|
||||||
score = result.get("score")
|
score = result.get("score")
|
||||||
post_id = result.get("post_id")
|
post_id = result.get("post_id")
|
||||||
|
tags_preview = result.get("tags_preview")
|
||||||
entry = {
|
entry = {
|
||||||
"post_id": post_id if isinstance(post_id, int) else None,
|
"post_id": post_id if isinstance(post_id, int) else None,
|
||||||
"score": float(score) if isinstance(score, (int, float)) else None,
|
"score": float(score) if isinstance(score, (int, float)) else None,
|
||||||
"preview_url": str(result.get("preview_url") or "")[:500] or None,
|
"preview_url": str(result.get("preview_url") or "")[:500] or None,
|
||||||
"rating": str(result.get("rating") or "")[:1] or None,
|
"rating": str(result.get("rating") or "")[:1] or None,
|
||||||
"md5": str(result.get("md5") or "")[:32] or None,
|
"md5": str(result.get("md5") or "")[:32] or None,
|
||||||
|
"score_total": (
|
||||||
|
result.get("score_total")
|
||||||
|
if isinstance(result.get("score_total"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"fav_count": (
|
||||||
|
result.get("fav_count")
|
||||||
|
if isinstance(result.get("fav_count"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"width": (
|
||||||
|
result.get("width")
|
||||||
|
if isinstance(result.get("width"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"height": (
|
||||||
|
result.get("height")
|
||||||
|
if isinstance(result.get("height"), int)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"tags_preview": (
|
||||||
|
[str(tag)[:100] for tag in tags_preview][:8]
|
||||||
|
if isinstance(tags_preview, list)
|
||||||
|
else []
|
||||||
|
),
|
||||||
}
|
}
|
||||||
if entry["post_id"] or entry["preview_url"]:
|
if entry["post_id"] or entry["preview_url"]:
|
||||||
cleaned.append(entry)
|
cleaned.append(entry)
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ Flow:
|
|||||||
import logging
|
import logging
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
@@ -31,20 +32,43 @@ from .serializers import TempUploadSerializer
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def complete_temp_upload(temp):
|
def complete_temp_upload(temp, download_url=None):
|
||||||
"""Move a staged file into the watched folder and index it."""
|
"""Index the upload into the library.
|
||||||
|
|
||||||
|
With a ``download_url`` the e621 original is fetched and the staged copy
|
||||||
|
is dropped; otherwise the staged file itself moves into the library.
|
||||||
|
"""
|
||||||
folder = Path(settings.WATCHED_FOLDER)
|
folder = Path(settings.WATCHED_FOLDER)
|
||||||
|
|
||||||
|
if download_url:
|
||||||
|
file_data = {}
|
||||||
|
if isinstance(temp.e621_data, dict):
|
||||||
|
file_data = temp.e621_data.get("file") or {}
|
||||||
|
extension = Path(urlparse(download_url).path).suffix
|
||||||
|
if not extension:
|
||||||
|
extension = Path(temp.original_filename).suffix
|
||||||
|
md5 = file_data.get("md5")
|
||||||
|
name = f"{md5}{extension}" if md5 else (temp.original_filename or "download")
|
||||||
|
destination = services.unique_destination(folder, name)
|
||||||
|
try:
|
||||||
|
services.download_file(download_url, destination)
|
||||||
|
except Exception:
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
raise
|
||||||
|
item, _, _ = services.index_file(destination, folder)
|
||||||
|
if temp.file:
|
||||||
|
temp.file.delete(save=False)
|
||||||
|
else:
|
||||||
if not temp.file:
|
if not temp.file:
|
||||||
raise ValueError("staged file is missing")
|
raise ValueError("staged file is missing")
|
||||||
|
|
||||||
destination = services.unique_destination(
|
destination = services.unique_destination(
|
||||||
folder, temp.original_filename or "upload"
|
folder, temp.original_filename or "upload"
|
||||||
)
|
)
|
||||||
with temp.file.open("rb") as source, open(destination, "wb") as target:
|
with temp.file.open("rb") as source, open(destination, "wb") as target:
|
||||||
shutil.copyfileobj(source, target)
|
shutil.copyfileobj(source, target)
|
||||||
|
|
||||||
item, _, _ = services.index_file(destination, folder)
|
item, _, _ = services.index_file(destination, folder)
|
||||||
temp.file.delete(save=False)
|
temp.file.delete(save=False)
|
||||||
|
|
||||||
temp.library_item = item
|
temp.library_item = item
|
||||||
temp.status = TempUpload.STATUS_COMPLETED
|
temp.status = TempUpload.STATUS_COMPLETED
|
||||||
|
|
||||||
@@ -184,6 +208,7 @@ class TempUploadViewSet(
|
|||||||
)
|
)
|
||||||
|
|
||||||
mode = str(request.data.get("mode") or "").strip()
|
mode = str(request.data.get("mode") or "").strip()
|
||||||
|
download_url = None
|
||||||
if mode == "link":
|
if mode == "link":
|
||||||
post = request.data.get("post")
|
post = request.data.get("post")
|
||||||
post_id = request.data.get("post_id")
|
post_id = request.data.get("post_id")
|
||||||
@@ -213,6 +238,17 @@ class TempUploadViewSet(
|
|||||||
notes = request.data.get("notes")
|
notes = request.data.get("notes")
|
||||||
if notes is not None:
|
if notes is not None:
|
||||||
temp.custom_notes = str(notes)
|
temp.custom_notes = str(notes)
|
||||||
|
|
||||||
|
# Prefer the e621 original; fall back to the staged file when the
|
||||||
|
# post has no URL or its file is byte-identical to the upload.
|
||||||
|
file_data = (trimmed or {}).get("file") or {}
|
||||||
|
candidate_url = str(
|
||||||
|
request.data.get("file_url") or file_data.get("url") or ""
|
||||||
|
).strip()
|
||||||
|
post_md5 = str(file_data.get("md5") or "").strip().lower()
|
||||||
|
if candidate_url and post_md5 and post_md5 == temp.md5.lower():
|
||||||
|
candidate_url = ""
|
||||||
|
download_url = candidate_url or None
|
||||||
elif mode == "custom":
|
elif mode == "custom":
|
||||||
rating = str(request.data.get("rating") or "")
|
rating = str(request.data.get("rating") or "")
|
||||||
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
|
temp.custom_rating = rating if rating in {"s", "q", "e"} else ""
|
||||||
@@ -227,7 +263,7 @@ class TempUploadViewSet(
|
|||||||
|
|
||||||
temp.save()
|
temp.save()
|
||||||
try:
|
try:
|
||||||
complete_temp_upload(temp)
|
complete_temp_upload(temp, download_url=download_url)
|
||||||
except Exception as exc: # noqa: BLE001 - report completion failures
|
except Exception as exc: # noqa: BLE001 - report completion failures
|
||||||
logger.exception("Could not complete staged upload %s", temp.id)
|
logger.exception("Could not complete staged upload %s", temp.id)
|
||||||
temp.status = TempUpload.STATUS_ERROR
|
temp.status = TempUpload.STATUS_ERROR
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import {
|
|||||||
fetchPost,
|
fetchPost,
|
||||||
fetchPostsByMd5,
|
fetchPostsByMd5,
|
||||||
iqdbSearch,
|
iqdbSearch,
|
||||||
|
RATING_LABELS,
|
||||||
} from "@/lib/e621";
|
} from "@/lib/e621";
|
||||||
import { formatBytes } from "@/lib/format";
|
import { formatBytes } from "@/lib/format";
|
||||||
import type {
|
import type {
|
||||||
@@ -45,6 +46,12 @@ const RESOLUTION_LABELS: Record<string, string> = {
|
|||||||
custom: "custom metadata",
|
custom: "custom metadata",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const ratingPill: Record<string, string> = {
|
||||||
|
s: "bg-ctp-green text-ctp-crust",
|
||||||
|
q: "bg-ctp-peach text-ctp-crust",
|
||||||
|
e: "bg-ctp-red text-ctp-crust",
|
||||||
|
};
|
||||||
|
|
||||||
const ratingChoices: { value: Rating; label: string; active: string }[] = [
|
const ratingChoices: { value: Rating; label: string; active: string }[] = [
|
||||||
{
|
{
|
||||||
value: "",
|
value: "",
|
||||||
@@ -205,6 +212,7 @@ function MetadataModal({
|
|||||||
const credentials = useE621((state) => state.credentials);
|
const credentials = useE621((state) => state.credentials);
|
||||||
const [tab, setTab] = useState<"link" | "custom">("link");
|
const [tab, setTab] = useState<"link" | "custom">("link");
|
||||||
const [postId, setPostId] = useState("");
|
const [postId, setPostId] = useState("");
|
||||||
|
const [selected, setSelected] = useState<E621IqdbCandidate | null>(null);
|
||||||
const [rating, setRating] = useState<Rating>(temp.custom_rating);
|
const [rating, setRating] = useState<Rating>(temp.custom_rating);
|
||||||
const [tags, setTags] = useState(temp.custom_tags.join(", "));
|
const [tags, setTags] = useState(temp.custom_tags.join(", "));
|
||||||
const [notes, setNotes] = useState(temp.custom_notes);
|
const [notes, setNotes] = useState(temp.custom_notes);
|
||||||
@@ -354,17 +362,39 @@ function MetadataModal({
|
|||||||
key={`${candidate.post_id}-${candidate.md5}`}
|
key={`${candidate.post_id}-${candidate.md5}`}
|
||||||
type="button"
|
type="button"
|
||||||
disabled={busy || !candidate.post_id}
|
disabled={busy || !candidate.post_id}
|
||||||
onClick={() => void linkCandidate(candidate)}
|
onClick={() => setSelected(candidate)}
|
||||||
className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle text-left transition hover:border-ctp-surface2 disabled:opacity-50"
|
className={cn(
|
||||||
|
"overflow-hidden rounded-md border bg-ctp-mantle text-left transition disabled:opacity-50",
|
||||||
|
selected?.post_id === candidate.post_id
|
||||||
|
? "border-ctp-mauve ring-2 ring-ctp-mauve/40"
|
||||||
|
: "border-ctp-surface0 hover:border-ctp-surface2",
|
||||||
|
)}
|
||||||
>
|
>
|
||||||
|
<span className="relative block aspect-square overflow-hidden">
|
||||||
{candidate.preview_url ? (
|
{candidate.preview_url ? (
|
||||||
<img
|
<img
|
||||||
src={candidate.preview_url}
|
src={candidate.preview_url}
|
||||||
alt={`Post #${candidate.post_id}`}
|
alt={`Post #${candidate.post_id}`}
|
||||||
loading="lazy"
|
loading="lazy"
|
||||||
className="aspect-square w-full object-cover"
|
className="h-full w-full object-cover"
|
||||||
/>
|
/>
|
||||||
|
) : (
|
||||||
|
<span className="flex h-full w-full items-center justify-center text-[10px] text-ctp-overlay0">
|
||||||
|
No preview
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{candidate.rating ? (
|
||||||
|
<span
|
||||||
|
className={cn(
|
||||||
|
"absolute left-1 top-1 rounded-full px-1 py-0.5 font-mono text-[9px] font-semibold uppercase",
|
||||||
|
ratingPill[candidate.rating] ??
|
||||||
|
"bg-ctp-surface1 text-ctp-text",
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{candidate.rating}
|
||||||
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
|
</span>
|
||||||
<span className="block px-1.5 py-1 font-mono text-[10px] text-ctp-overlay0">
|
<span className="block px-1.5 py-1 font-mono text-[10px] text-ctp-overlay0">
|
||||||
#{candidate.post_id}
|
#{candidate.post_id}
|
||||||
{candidate.score !== null
|
{candidate.score !== null
|
||||||
@@ -374,6 +404,62 @@ function MetadataModal({
|
|||||||
</button>
|
</button>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{selected ? (
|
||||||
|
<div className="mt-2 rounded-md border border-ctp-surface0 bg-ctp-surface0/40 p-3">
|
||||||
|
<p className="font-mono text-xs text-ctp-subtext1">
|
||||||
|
Post #{selected.post_id}
|
||||||
|
</p>
|
||||||
|
<p className="mt-1 text-xs text-ctp-overlay0">
|
||||||
|
{selected.rating
|
||||||
|
? (RATING_LABELS[selected.rating] ?? selected.rating)
|
||||||
|
: "Unknown rating"}
|
||||||
|
{selected.score_total !== null &&
|
||||||
|
selected.score_total !== undefined
|
||||||
|
? ` · ▲ ${selected.score_total}`
|
||||||
|
: ""}
|
||||||
|
{selected.fav_count !== null &&
|
||||||
|
selected.fav_count !== undefined
|
||||||
|
? ` · ${selected.fav_count} favs`
|
||||||
|
: ""}
|
||||||
|
{selected.width && selected.height
|
||||||
|
? ` · ${selected.width}×${selected.height}`
|
||||||
|
: ""}
|
||||||
|
</p>
|
||||||
|
{selected.tags_preview &&
|
||||||
|
selected.tags_preview.length > 0 ? (
|
||||||
|
<div className="mt-2 flex flex-wrap gap-1">
|
||||||
|
{selected.tags_preview.map((tag) => (
|
||||||
|
<span
|
||||||
|
key={tag}
|
||||||
|
className="rounded border border-ctp-surface1 px-1 py-0.5 font-mono text-[10px] text-ctp-subtext0"
|
||||||
|
>
|
||||||
|
{tag}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
<div className="mt-3 flex items-center gap-3">
|
||||||
|
<Button
|
||||||
|
onClick={() => void linkCandidate(selected)}
|
||||||
|
disabled={busy || !selected.post_id}
|
||||||
|
>
|
||||||
|
Link selected post
|
||||||
|
</Button>
|
||||||
|
<Link
|
||||||
|
to={`/detail/${selected.post_id}`}
|
||||||
|
target="_blank"
|
||||||
|
className="text-xs text-ctp-blue hover:underline"
|
||||||
|
>
|
||||||
|
View post
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
<p className="mt-2 text-[11px] text-ctp-overlay0">
|
||||||
|
Linking downloads the post's file into the library and
|
||||||
|
drops this staged upload.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<p className="text-xs text-ctp-overlay0">
|
<p className="text-xs text-ctp-overlay0">
|
||||||
|
|||||||
@@ -322,6 +322,26 @@ export interface E621IqdbResult {
|
|||||||
preview_url: string | null;
|
preview_url: string | null;
|
||||||
rating: string | null;
|
rating: string | null;
|
||||||
md5: string | null;
|
md5: string | null;
|
||||||
|
score_total: number | null;
|
||||||
|
fav_count: number | null;
|
||||||
|
width: number | null;
|
||||||
|
height: number | null;
|
||||||
|
tags_preview: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
function flattenTagPreview(tags: unknown, limit = 8): string[] {
|
||||||
|
if (!tags || typeof tags !== "object") return [];
|
||||||
|
const out: string[] = [];
|
||||||
|
for (const value of Object.values(tags as Record<string, unknown>)) {
|
||||||
|
if (!Array.isArray(value)) continue;
|
||||||
|
for (const tag of value) {
|
||||||
|
if (typeof tag === "string" && !out.includes(tag)) {
|
||||||
|
out.push(tag);
|
||||||
|
if (out.length >= limit) return out;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeIqdbEntry(entry: unknown): E621IqdbResult | null {
|
function normalizeIqdbEntry(entry: unknown): E621IqdbResult | null {
|
||||||
@@ -344,12 +364,18 @@ function normalizeIqdbEntry(entry: unknown): E621IqdbResult | null {
|
|||||||
: typeof post.id === "number"
|
: typeof post.id === "number"
|
||||||
? post.id
|
? post.id
|
||||||
: null;
|
: null;
|
||||||
|
const scoreTotal = post.score;
|
||||||
return {
|
return {
|
||||||
post_id: postId,
|
post_id: postId,
|
||||||
score: typeof record.score === "number" ? record.score : null,
|
score: typeof record.score === "number" ? record.score : null,
|
||||||
preview_url: typeof preview.url === "string" ? preview.url : null,
|
preview_url: typeof preview.url === "string" ? preview.url : null,
|
||||||
rating: typeof post.rating === "string" ? post.rating : null,
|
rating: typeof post.rating === "string" ? post.rating : null,
|
||||||
md5: typeof file.md5 === "string" ? file.md5 : null,
|
md5: typeof file.md5 === "string" ? file.md5 : null,
|
||||||
|
score_total: typeof scoreTotal === "number" ? scoreTotal : null,
|
||||||
|
fav_count: typeof post.fav_count === "number" ? post.fav_count : null,
|
||||||
|
width: typeof file.width === "number" ? file.width : null,
|
||||||
|
height: typeof file.height === "number" ? file.height : null,
|
||||||
|
tags_preview: flattenTagPreview(post.tags),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,13 +397,44 @@ export async function iqdbSearch(
|
|||||||
: Array.isArray((data as { matches?: unknown[] })?.matches)
|
: Array.isArray((data as { matches?: unknown[] })?.matches)
|
||||||
? (data as { matches: unknown[] }).matches
|
? (data as { matches: unknown[] }).matches
|
||||||
: [];
|
: [];
|
||||||
return list
|
const candidates = list
|
||||||
.map(normalizeIqdbEntry)
|
.map(normalizeIqdbEntry)
|
||||||
.filter(
|
.filter(
|
||||||
(entry): entry is E621IqdbResult =>
|
(entry): entry is E621IqdbResult =>
|
||||||
entry !== null && (entry.post_id !== null || entry.preview_url !== null),
|
entry !== null && (entry.post_id !== null || entry.preview_url !== null),
|
||||||
)
|
)
|
||||||
.slice(0, 10);
|
.slice(0, 10);
|
||||||
|
|
||||||
|
// The IQDB payload carries no preview/file data; enrich with one batched
|
||||||
|
// post lookup so candidates can show what they actually are.
|
||||||
|
const ids = candidates
|
||||||
|
.map((candidate) => candidate.post_id)
|
||||||
|
.filter((id): id is number => typeof id === "number");
|
||||||
|
if (ids.length > 0) {
|
||||||
|
try {
|
||||||
|
const posts = await fetchPostsByIds(credentials, ids, signal);
|
||||||
|
const byId = new Map(posts.map((post) => [post.id, post]));
|
||||||
|
return candidates.map((candidate) => {
|
||||||
|
const post =
|
||||||
|
candidate.post_id !== null ? byId.get(candidate.post_id) : undefined;
|
||||||
|
if (!post) return candidate;
|
||||||
|
return {
|
||||||
|
...candidate,
|
||||||
|
preview_url: post.preview.url ?? candidate.preview_url,
|
||||||
|
rating: post.rating ?? candidate.rating,
|
||||||
|
md5: post.file.md5 ?? candidate.md5,
|
||||||
|
score_total: post.score.total,
|
||||||
|
fav_count: post.fav_count,
|
||||||
|
width: post.file.width,
|
||||||
|
height: post.file.height,
|
||||||
|
tags_preview: flattenTagPreview(post.tags),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Keep the base candidates when the enrichment request fails.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return candidates;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function postIsBlacklisted(
|
export function postIsBlacklisted(
|
||||||
|
|||||||
@@ -113,6 +113,11 @@ export interface E621IqdbCandidate {
|
|||||||
preview_url: string | null;
|
preview_url: string | null;
|
||||||
rating: string | null;
|
rating: string | null;
|
||||||
md5: string | null;
|
md5: string | null;
|
||||||
|
score_total?: number | null;
|
||||||
|
fav_count?: number | null;
|
||||||
|
width?: number | null;
|
||||||
|
height?: number | null;
|
||||||
|
tags_preview?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TempUpload {
|
export interface TempUpload {
|
||||||
|
|||||||
Reference in New Issue
Block a user