Upload board: dismiss all, real previews for indexed records

- 'dismiss all' clears every indexed record at once
- Indexed cards show the actual file preview: completed records now get a
  signed library media URL (raw for images, thumbnail for videos) so
  <img>/<video> tags can load it, including items hidden from guests
- Media raw/thumbnail endpoints accept the signature for anonymous
  requests and fall back to the normal guest-filtered path otherwise
- Guest blacklist keeps a persistent Redis mirror: an expired TTL or an
  unreachable e621 keeps the last successful list instead of falling
  back to the small local list
This commit is contained in:
2026-09-17 11:29:56 -05:00
parent b71ec729e0
commit 1086beb974
6 changed files with 117 additions and 19 deletions
+15 -2
View File
@@ -20,6 +20,7 @@ from django.core.cache import cache
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
CACHE_KEY = "j621.guest_blacklist" CACHE_KEY = "j621.guest_blacklist"
MIRROR_KEY = "j621.guest_blacklist.mirror"
def parse_blacklist_patterns(lines): def parse_blacklist_patterns(lines):
@@ -37,10 +38,18 @@ def parse_blacklist_patterns(lines):
def cached_guest_blacklist(): def cached_guest_blacklist():
"""Raw mirrored blacklist lines. Never hits the network.""" """Raw mirrored blacklist lines. Never hits the network.
Prefers the TTL'd entry, falls back to the persistent mirror (last
successful e621 fetch) and only then to the local fallback list, so an
expired TTL never degrades filtering.
"""
stored = cache.get(CACHE_KEY) stored = cache.get(CACHE_KEY)
if stored is not None: if stored is not None:
return list(stored) return list(stored)
mirror = cache.get(MIRROR_KEY)
if mirror is not None:
return list(mirror)
return list(settings.GUEST_BLACKLIST_FALLBACK) return list(settings.GUEST_BLACKLIST_FALLBACK)
@@ -101,7 +110,11 @@ def refresh_guest_blacklist():
lines = fetch_e621_default_blacklist() lines = fetch_e621_default_blacklist()
used_fallback = lines is None used_fallback = lines is None
if used_fallback: if used_fallback:
lines = list(settings.GUEST_BLACKLIST_FALLBACK) # Keep the last successful mirror (or the local list) rather than
# degrading guest filtering when e621 is unreachable.
lines = cache.get(MIRROR_KEY) or list(settings.GUEST_BLACKLIST_FALLBACK)
else:
cache.set(MIRROR_KEY, lines, timeout=None)
cache.set(CACHE_KEY, lines, settings.GUEST_BLACKLIST_TTL) cache.set(CACHE_KEY, lines, settings.GUEST_BLACKLIST_TTL)
patterns = parse_blacklist_patterns(lines) patterns = parse_blacklist_patterns(lines)
+35 -4
View File
@@ -1,10 +1,11 @@
import os import os
from pathlib import Path
from django.core import signing from django.core import signing
from rest_framework import serializers from rest_framework import serializers
from .models import MediaItem, MediaLocation, TempUpload from .models import MediaItem, MediaLocation, TempUpload
from .services import UPLOAD_FILE_SALT, VIDEO_EXTENSIONS from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
class MediaLocationSerializer(serializers.ModelSerializer): class MediaLocationSerializer(serializers.ModelSerializer):
@@ -99,6 +100,7 @@ class TempUploadSerializer(serializers.ModelSerializer):
temp_id = serializers.UUIDField(source="id", read_only=True) temp_id = serializers.UUIDField(source="id", read_only=True)
library_j_id = serializers.SerializerMethodField() library_j_id = serializers.SerializerMethodField()
file_url = serializers.SerializerMethodField() file_url = serializers.SerializerMethodField()
preview_url = serializers.SerializerMethodField()
class Meta: class Meta:
model = TempUpload model = TempUpload
@@ -117,11 +119,19 @@ class TempUploadSerializer(serializers.ModelSerializer):
"iqdb_data", "iqdb_data",
"library_j_id", "library_j_id",
"file_url", "file_url",
"preview_url",
"created_at", "created_at",
"updated_at", "updated_at",
] ]
read_only_fields = fields read_only_fields = fields
def _request_user(self):
request = self.context.get("request")
user = getattr(request, "user", None)
if user is None or not getattr(user, "is_authenticated", False):
return None
return user
def get_library_j_id(self, obj): def get_library_j_id(self, obj):
return f"J-{obj.library_item_id}" if obj.library_item_id else None return f"J-{obj.library_item_id}" if obj.library_item_id else None
@@ -129,12 +139,33 @@ class TempUploadSerializer(serializers.ModelSerializer):
"""Signed URL so <img>/<video> tags can fetch the staged file.""" """Signed URL so <img>/<video> tags can fetch the staged file."""
if not obj.file: if not obj.file:
return None return None
request = self.context.get("request") user = self._request_user()
user = getattr(request, "user", None) if user is None:
if user is None or not getattr(user, "is_authenticated", False):
return None return None
signature = signing.dumps( signature = signing.dumps(
{"temp": str(obj.id), "user": user.id}, {"temp": str(obj.id), "user": user.id},
salt=UPLOAD_FILE_SALT, salt=UPLOAD_FILE_SALT,
) )
return f"/api/uploads/{obj.id}/file/?sig={signature}" return f"/api/uploads/{obj.id}/file/?sig={signature}"
def get_preview_url(self, obj):
"""A URL an <img>/<video> tag can load: staged file or library item."""
user = self._request_user()
if user is None:
return None
if obj.file:
return self.get_file_url(obj)
item = obj.library_item
if item is None:
return None
location = item.locations.first()
is_video = (
location is not None
and Path(location.rel_path).suffix.lower() in VIDEO_EXTENSIONS
)
action = "thumbnail" if is_video else "raw"
signature = signing.dumps(
{"item": item.id, "user": user.id, "action": action},
salt=MEDIA_FILE_SALT,
)
return f"/api/files/J-{item.id}/{action}/?sig={signature}"
+1
View File
@@ -25,6 +25,7 @@ ALLOWED_EXTENSIONS = {
} }
VIDEO_EXTENSIONS = {".mp4", ".webm"} VIDEO_EXTENSIONS = {".mp4", ".webm"}
UPLOAD_FILE_SALT = "j621.upload-file" UPLOAD_FILE_SALT = "j621.upload-file"
MEDIA_FILE_SALT = "j621.media-file"
CHUNK_SIZE = 1024 * 1024 CHUNK_SIZE = 1024 * 1024
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$") RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
+26 -2
View File
@@ -3,6 +3,7 @@ from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from django.conf import settings from django.conf import settings
from django.core import signing
from django.db.models import Min from django.db.models import Min
from django.http import Http404 from django.http import Http404
from django.shortcuts import get_object_or_404 from django.shortcuts import get_object_or_404
@@ -67,9 +68,32 @@ class MediaItemViewSet(
self.check_object_permissions(self.request, obj) self.check_object_permissions(self.request, obj)
return obj return obj
def _signed_media_item(self, request, action_name):
"""Signed media URLs let <img>/<video> tags bypass guest filtering."""
if request.user.is_authenticated:
return None
signature = request.query_params.get("sig")
if not signature:
return None
try:
payload = signing.loads(
signature, salt=services.MEDIA_FILE_SALT, max_age=86400
)
except signing.BadSignature:
return None
if payload.get("action") != action_name:
return None
return MediaItem.objects.filter(pk=payload.get("item")).first()
def _media_object(self, request, action_name):
item = self._signed_media_item(request, action_name)
if item is not None:
return item
return self.get_object()
@action(detail=True, methods=["get"]) @action(detail=True, methods=["get"])
def raw(self, request, pk=None): def raw(self, request, pk=None):
item = self.get_object() item = self._media_object(request, "raw")
location = item.locations.first() location = item.locations.first()
if location is None: if location is None:
return Response( return Response(
@@ -82,7 +106,7 @@ class MediaItemViewSet(
@action(detail=True, methods=["get"]) @action(detail=True, methods=["get"])
def thumbnail(self, request, pk=None): def thumbnail(self, request, pk=None):
item = self.get_object() item = self._media_object(request, "thumbnail")
location = item.locations.first() location = item.locations.first()
if location is None: if location is None:
return Response( return Response(
+39 -11
View File
@@ -4,7 +4,7 @@ import {
UploadCloud, UploadCloud,
X, X,
} from "lucide-react"; } from "lucide-react";
import { useRef, useState, type DragEvent } from "react"; import { useRef, useState, type DragEvent, type ReactNode } from "react";
import { Link } from "react-router-dom"; import { Link } from "react-router-dom";
import { Button, EmptyState, Spinner, inputClass } from "@/components/ui"; import { Button, EmptyState, Spinner, inputClass } from "@/components/ui";
@@ -102,14 +102,15 @@ function TempCard({
onDismiss: () => void; onDismiss: () => void;
}) { }) {
const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename); const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename);
const preview = temp.preview_url ?? temp.file_url;
return ( return (
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5"> <div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle"> <div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
{temp.file_url ? ( {preview ? (
isVideo ? ( isVideo ? (
<video <video
src={temp.file_url} src={preview}
muted muted
playsInline playsInline
preload="metadata" preload="metadata"
@@ -117,7 +118,7 @@ function TempCard({
/> />
) : ( ) : (
<img <img
src={temp.file_url} src={preview}
alt={temp.original_filename} alt={temp.original_filename}
loading="lazy" loading="lazy"
className="h-full w-full object-cover" className="h-full w-full object-cover"
@@ -125,7 +126,7 @@ function TempCard({
) )
) : ( ) : (
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0"> <div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
{temp.status === "completed" ? "Indexed" : "No preview"} No preview
</div> </div>
)} )}
{temp.status === "visual_match" && temp.iqdb_data?.length ? ( {temp.status === "visual_match" && temp.iqdb_data?.length ? (
@@ -303,17 +304,17 @@ function MetadataModal({
<div className="mt-4 grid gap-4 sm:grid-cols-[200px_minmax(0,1fr)]"> <div className="mt-4 grid gap-4 sm:grid-cols-[200px_minmax(0,1fr)]">
<div className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle"> <div className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle">
{temp.file_url ? ( {(temp.preview_url ?? temp.file_url) ? (
isVideo ? ( isVideo ? (
<video <video
src={temp.file_url} src={(temp.preview_url ?? temp.file_url) as string}
controls controls
muted muted
className="max-h-52 w-full object-contain" className="max-h-52 w-full object-contain"
/> />
) : ( ) : (
<img <img
src={temp.file_url} src={(temp.preview_url ?? temp.file_url) as string}
alt={temp.original_filename} alt={temp.original_filename}
className="max-h-52 w-full object-contain" className="max-h-52 w-full object-contain"
/> />
@@ -715,6 +716,15 @@ export default function UploadPage() {
); );
} }
function dismissAll(items: TempUpload[]) {
if (items.length === 0) return;
void Promise.allSettled(
items.map((temp) =>
api(`/api/uploads/${temp.temp_id}/`, { method: "DELETE" }),
),
).then(() => invalidateUploads());
}
function clearFinished() { function clearFinished() {
setEntries((current) => current.filter((entry) => entry.status !== "done")); setEntries((current) => current.filter((entry) => entry.status !== "done"));
} }
@@ -737,12 +747,16 @@ export default function UploadPage() {
title: string, title: string,
items: TempUpload[], items: TempUpload[],
emptyText: string, emptyText: string,
action?: ReactNode,
) => ( ) => (
<section className="flex min-w-0 flex-col gap-3"> <section className="flex min-w-0 flex-col gap-3">
<header className="flex items-center justify-between gap-2"> <header className="flex items-center justify-between gap-2">
<h2 className="text-sm font-semibold text-ctp-subtext1">{title}</h2> <h2 className="text-sm font-semibold text-ctp-subtext1">{title}</h2>
<span className="font-mono text-[11px] text-ctp-overlay0"> <span className="flex items-center gap-2">
{items.length} {action}
<span className="font-mono text-[11px] text-ctp-overlay0">
{items.length}
</span>
</span> </span>
</header> </header>
{items.length === 0 ? ( {items.length === 0 ? (
@@ -888,7 +902,21 @@ export default function UploadPage() {
visual, visual,
"No IQDB matches right now.", "No IQDB matches right now.",
)} )}
{column("Auto-uploaded & Indexed", completed, "Nothing indexed yet.")} {column(
"Auto-uploaded & Indexed",
completed,
"Nothing indexed yet.",
completed.length > 0 ? (
<button
type="button"
onClick={() => dismissAll(completed)}
title="Dismiss every indexed record"
className="rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
>
dismiss all
</button>
) : null,
)}
</div> </div>
{modalFor ? ( {modalFor ? (
+1
View File
@@ -135,6 +135,7 @@ export interface TempUpload {
iqdb_data: E621IqdbCandidate[] | null; iqdb_data: E621IqdbCandidate[] | null;
library_j_id: string | null; library_j_id: string | null;
file_url: string | null; file_url: string | null;
preview_url: string | null;
created_at: string; created_at: string;
updated_at: string; updated_at: string;
} }