Upload board: dismiss all, real previews for indexed records
- 'dismiss all' clears every indexed record at once - Indexed cards show the actual file preview: completed records now get a signed library media URL (raw for images, thumbnail for videos) so <img>/<video> tags can load it, including items hidden from guests - Media raw/thumbnail endpoints accept the signature for anonymous requests and fall back to the normal guest-filtered path otherwise - Guest blacklist keeps a persistent Redis mirror: an expired TTL or an unreachable e621 keeps the last successful list instead of falling back to the small local list
This commit is contained in:
@@ -20,6 +20,7 @@ from django.core.cache import cache
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
CACHE_KEY = "j621.guest_blacklist"
|
CACHE_KEY = "j621.guest_blacklist"
|
||||||
|
MIRROR_KEY = "j621.guest_blacklist.mirror"
|
||||||
|
|
||||||
|
|
||||||
def parse_blacklist_patterns(lines):
|
def parse_blacklist_patterns(lines):
|
||||||
@@ -37,10 +38,18 @@ def parse_blacklist_patterns(lines):
|
|||||||
|
|
||||||
|
|
||||||
def cached_guest_blacklist():
|
def cached_guest_blacklist():
|
||||||
"""Raw mirrored blacklist lines. Never hits the network."""
|
"""Raw mirrored blacklist lines. Never hits the network.
|
||||||
|
|
||||||
|
Prefers the TTL'd entry, falls back to the persistent mirror (last
|
||||||
|
successful e621 fetch) and only then to the local fallback list, so an
|
||||||
|
expired TTL never degrades filtering.
|
||||||
|
"""
|
||||||
stored = cache.get(CACHE_KEY)
|
stored = cache.get(CACHE_KEY)
|
||||||
if stored is not None:
|
if stored is not None:
|
||||||
return list(stored)
|
return list(stored)
|
||||||
|
mirror = cache.get(MIRROR_KEY)
|
||||||
|
if mirror is not None:
|
||||||
|
return list(mirror)
|
||||||
return list(settings.GUEST_BLACKLIST_FALLBACK)
|
return list(settings.GUEST_BLACKLIST_FALLBACK)
|
||||||
|
|
||||||
|
|
||||||
@@ -101,7 +110,11 @@ def refresh_guest_blacklist():
|
|||||||
lines = fetch_e621_default_blacklist()
|
lines = fetch_e621_default_blacklist()
|
||||||
used_fallback = lines is None
|
used_fallback = lines is None
|
||||||
if used_fallback:
|
if used_fallback:
|
||||||
lines = list(settings.GUEST_BLACKLIST_FALLBACK)
|
# Keep the last successful mirror (or the local list) rather than
|
||||||
|
# degrading guest filtering when e621 is unreachable.
|
||||||
|
lines = cache.get(MIRROR_KEY) or list(settings.GUEST_BLACKLIST_FALLBACK)
|
||||||
|
else:
|
||||||
|
cache.set(MIRROR_KEY, lines, timeout=None)
|
||||||
cache.set(CACHE_KEY, lines, settings.GUEST_BLACKLIST_TTL)
|
cache.set(CACHE_KEY, lines, settings.GUEST_BLACKLIST_TTL)
|
||||||
|
|
||||||
patterns = parse_blacklist_patterns(lines)
|
patterns = parse_blacklist_patterns(lines)
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from django.core import signing
|
from django.core import signing
|
||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
from .models import MediaItem, MediaLocation, TempUpload
|
from .models import MediaItem, MediaLocation, TempUpload
|
||||||
from .services import UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
|
from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
class MediaLocationSerializer(serializers.ModelSerializer):
|
class MediaLocationSerializer(serializers.ModelSerializer):
|
||||||
@@ -99,6 +100,7 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
temp_id = serializers.UUIDField(source="id", read_only=True)
|
temp_id = serializers.UUIDField(source="id", read_only=True)
|
||||||
library_j_id = serializers.SerializerMethodField()
|
library_j_id = serializers.SerializerMethodField()
|
||||||
file_url = serializers.SerializerMethodField()
|
file_url = serializers.SerializerMethodField()
|
||||||
|
preview_url = serializers.SerializerMethodField()
|
||||||
|
|
||||||
class Meta:
|
class Meta:
|
||||||
model = TempUpload
|
model = TempUpload
|
||||||
@@ -117,11 +119,19 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
"iqdb_data",
|
"iqdb_data",
|
||||||
"library_j_id",
|
"library_j_id",
|
||||||
"file_url",
|
"file_url",
|
||||||
|
"preview_url",
|
||||||
"created_at",
|
"created_at",
|
||||||
"updated_at",
|
"updated_at",
|
||||||
]
|
]
|
||||||
read_only_fields = fields
|
read_only_fields = fields
|
||||||
|
|
||||||
|
def _request_user(self):
|
||||||
|
request = self.context.get("request")
|
||||||
|
user = getattr(request, "user", None)
|
||||||
|
if user is None or not getattr(user, "is_authenticated", False):
|
||||||
|
return None
|
||||||
|
return user
|
||||||
|
|
||||||
def get_library_j_id(self, obj):
|
def get_library_j_id(self, obj):
|
||||||
return f"J-{obj.library_item_id}" if obj.library_item_id else None
|
return f"J-{obj.library_item_id}" if obj.library_item_id else None
|
||||||
|
|
||||||
@@ -129,12 +139,33 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
"""Signed URL so <img>/<video> tags can fetch the staged file."""
|
"""Signed URL so <img>/<video> tags can fetch the staged file."""
|
||||||
if not obj.file:
|
if not obj.file:
|
||||||
return None
|
return None
|
||||||
request = self.context.get("request")
|
user = self._request_user()
|
||||||
user = getattr(request, "user", None)
|
if user is None:
|
||||||
if user is None or not getattr(user, "is_authenticated", False):
|
|
||||||
return None
|
return None
|
||||||
signature = signing.dumps(
|
signature = signing.dumps(
|
||||||
{"temp": str(obj.id), "user": user.id},
|
{"temp": str(obj.id), "user": user.id},
|
||||||
salt=UPLOAD_FILE_SALT,
|
salt=UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
return f"/api/uploads/{obj.id}/file/?sig={signature}"
|
return f"/api/uploads/{obj.id}/file/?sig={signature}"
|
||||||
|
|
||||||
|
def get_preview_url(self, obj):
|
||||||
|
"""A URL an <img>/<video> tag can load: staged file or library item."""
|
||||||
|
user = self._request_user()
|
||||||
|
if user is None:
|
||||||
|
return None
|
||||||
|
if obj.file:
|
||||||
|
return self.get_file_url(obj)
|
||||||
|
item = obj.library_item
|
||||||
|
if item is None:
|
||||||
|
return None
|
||||||
|
location = item.locations.first()
|
||||||
|
is_video = (
|
||||||
|
location is not None
|
||||||
|
and Path(location.rel_path).suffix.lower() in VIDEO_EXTENSIONS
|
||||||
|
)
|
||||||
|
action = "thumbnail" if is_video else "raw"
|
||||||
|
signature = signing.dumps(
|
||||||
|
{"item": item.id, "user": user.id, "action": action},
|
||||||
|
salt=MEDIA_FILE_SALT,
|
||||||
|
)
|
||||||
|
return f"/api/files/J-{item.id}/{action}/?sig={signature}"
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ ALLOWED_EXTENSIONS = {
|
|||||||
}
|
}
|
||||||
VIDEO_EXTENSIONS = {".mp4", ".webm"}
|
VIDEO_EXTENSIONS = {".mp4", ".webm"}
|
||||||
UPLOAD_FILE_SALT = "j621.upload-file"
|
UPLOAD_FILE_SALT = "j621.upload-file"
|
||||||
|
MEDIA_FILE_SALT = "j621.media-file"
|
||||||
CHUNK_SIZE = 1024 * 1024
|
CHUNK_SIZE = 1024 * 1024
|
||||||
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from pathlib import Path
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from django.core import signing
|
||||||
from django.db.models import Min
|
from django.db.models import Min
|
||||||
from django.http import Http404
|
from django.http import Http404
|
||||||
from django.shortcuts import get_object_or_404
|
from django.shortcuts import get_object_or_404
|
||||||
@@ -67,9 +68,32 @@ class MediaItemViewSet(
|
|||||||
self.check_object_permissions(self.request, obj)
|
self.check_object_permissions(self.request, obj)
|
||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
def _signed_media_item(self, request, action_name):
|
||||||
|
"""Signed media URLs let <img>/<video> tags bypass guest filtering."""
|
||||||
|
if request.user.is_authenticated:
|
||||||
|
return None
|
||||||
|
signature = request.query_params.get("sig")
|
||||||
|
if not signature:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
payload = signing.loads(
|
||||||
|
signature, salt=services.MEDIA_FILE_SALT, max_age=86400
|
||||||
|
)
|
||||||
|
except signing.BadSignature:
|
||||||
|
return None
|
||||||
|
if payload.get("action") != action_name:
|
||||||
|
return None
|
||||||
|
return MediaItem.objects.filter(pk=payload.get("item")).first()
|
||||||
|
|
||||||
|
def _media_object(self, request, action_name):
|
||||||
|
item = self._signed_media_item(request, action_name)
|
||||||
|
if item is not None:
|
||||||
|
return item
|
||||||
|
return self.get_object()
|
||||||
|
|
||||||
@action(detail=True, methods=["get"])
|
@action(detail=True, methods=["get"])
|
||||||
def raw(self, request, pk=None):
|
def raw(self, request, pk=None):
|
||||||
item = self.get_object()
|
item = self._media_object(request, "raw")
|
||||||
location = item.locations.first()
|
location = item.locations.first()
|
||||||
if location is None:
|
if location is None:
|
||||||
return Response(
|
return Response(
|
||||||
@@ -82,7 +106,7 @@ class MediaItemViewSet(
|
|||||||
|
|
||||||
@action(detail=True, methods=["get"])
|
@action(detail=True, methods=["get"])
|
||||||
def thumbnail(self, request, pk=None):
|
def thumbnail(self, request, pk=None):
|
||||||
item = self.get_object()
|
item = self._media_object(request, "thumbnail")
|
||||||
location = item.locations.first()
|
location = item.locations.first()
|
||||||
if location is None:
|
if location is None:
|
||||||
return Response(
|
return Response(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import {
|
|||||||
UploadCloud,
|
UploadCloud,
|
||||||
X,
|
X,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import { useRef, useState, type DragEvent } from "react";
|
import { useRef, useState, type DragEvent, type ReactNode } from "react";
|
||||||
import { Link } from "react-router-dom";
|
import { Link } from "react-router-dom";
|
||||||
|
|
||||||
import { Button, EmptyState, Spinner, inputClass } from "@/components/ui";
|
import { Button, EmptyState, Spinner, inputClass } from "@/components/ui";
|
||||||
@@ -102,14 +102,15 @@ function TempCard({
|
|||||||
onDismiss: () => void;
|
onDismiss: () => void;
|
||||||
}) {
|
}) {
|
||||||
const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename);
|
const isVideo = /\.(mp4|webm)$/i.test(temp.original_filename);
|
||||||
|
const preview = temp.preview_url ?? temp.file_url;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
|
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
|
||||||
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
|
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
|
||||||
{temp.file_url ? (
|
{preview ? (
|
||||||
isVideo ? (
|
isVideo ? (
|
||||||
<video
|
<video
|
||||||
src={temp.file_url}
|
src={preview}
|
||||||
muted
|
muted
|
||||||
playsInline
|
playsInline
|
||||||
preload="metadata"
|
preload="metadata"
|
||||||
@@ -117,7 +118,7 @@ function TempCard({
|
|||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<img
|
<img
|
||||||
src={temp.file_url}
|
src={preview}
|
||||||
alt={temp.original_filename}
|
alt={temp.original_filename}
|
||||||
loading="lazy"
|
loading="lazy"
|
||||||
className="h-full w-full object-cover"
|
className="h-full w-full object-cover"
|
||||||
@@ -125,7 +126,7 @@ function TempCard({
|
|||||||
)
|
)
|
||||||
) : (
|
) : (
|
||||||
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
|
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
|
||||||
{temp.status === "completed" ? "Indexed" : "No preview"}
|
No preview
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{temp.status === "visual_match" && temp.iqdb_data?.length ? (
|
{temp.status === "visual_match" && temp.iqdb_data?.length ? (
|
||||||
@@ -303,17 +304,17 @@ function MetadataModal({
|
|||||||
|
|
||||||
<div className="mt-4 grid gap-4 sm:grid-cols-[200px_minmax(0,1fr)]">
|
<div className="mt-4 grid gap-4 sm:grid-cols-[200px_minmax(0,1fr)]">
|
||||||
<div className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle">
|
<div className="overflow-hidden rounded-md border border-ctp-surface0 bg-ctp-mantle">
|
||||||
{temp.file_url ? (
|
{(temp.preview_url ?? temp.file_url) ? (
|
||||||
isVideo ? (
|
isVideo ? (
|
||||||
<video
|
<video
|
||||||
src={temp.file_url}
|
src={(temp.preview_url ?? temp.file_url) as string}
|
||||||
controls
|
controls
|
||||||
muted
|
muted
|
||||||
className="max-h-52 w-full object-contain"
|
className="max-h-52 w-full object-contain"
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<img
|
<img
|
||||||
src={temp.file_url}
|
src={(temp.preview_url ?? temp.file_url) as string}
|
||||||
alt={temp.original_filename}
|
alt={temp.original_filename}
|
||||||
className="max-h-52 w-full object-contain"
|
className="max-h-52 w-full object-contain"
|
||||||
/>
|
/>
|
||||||
@@ -715,6 +716,15 @@ export default function UploadPage() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function dismissAll(items: TempUpload[]) {
|
||||||
|
if (items.length === 0) return;
|
||||||
|
void Promise.allSettled(
|
||||||
|
items.map((temp) =>
|
||||||
|
api(`/api/uploads/${temp.temp_id}/`, { method: "DELETE" }),
|
||||||
|
),
|
||||||
|
).then(() => invalidateUploads());
|
||||||
|
}
|
||||||
|
|
||||||
function clearFinished() {
|
function clearFinished() {
|
||||||
setEntries((current) => current.filter((entry) => entry.status !== "done"));
|
setEntries((current) => current.filter((entry) => entry.status !== "done"));
|
||||||
}
|
}
|
||||||
@@ -737,13 +747,17 @@ export default function UploadPage() {
|
|||||||
title: string,
|
title: string,
|
||||||
items: TempUpload[],
|
items: TempUpload[],
|
||||||
emptyText: string,
|
emptyText: string,
|
||||||
|
action?: ReactNode,
|
||||||
) => (
|
) => (
|
||||||
<section className="flex min-w-0 flex-col gap-3">
|
<section className="flex min-w-0 flex-col gap-3">
|
||||||
<header className="flex items-center justify-between gap-2">
|
<header className="flex items-center justify-between gap-2">
|
||||||
<h2 className="text-sm font-semibold text-ctp-subtext1">{title}</h2>
|
<h2 className="text-sm font-semibold text-ctp-subtext1">{title}</h2>
|
||||||
|
<span className="flex items-center gap-2">
|
||||||
|
{action}
|
||||||
<span className="font-mono text-[11px] text-ctp-overlay0">
|
<span className="font-mono text-[11px] text-ctp-overlay0">
|
||||||
{items.length}
|
{items.length}
|
||||||
</span>
|
</span>
|
||||||
|
</span>
|
||||||
</header>
|
</header>
|
||||||
{items.length === 0 ? (
|
{items.length === 0 ? (
|
||||||
<p className="rounded-lg border border-dashed border-ctp-surface1 px-3 py-6 text-center text-xs text-ctp-overlay0">
|
<p className="rounded-lg border border-dashed border-ctp-surface1 px-3 py-6 text-center text-xs text-ctp-overlay0">
|
||||||
@@ -888,7 +902,21 @@ export default function UploadPage() {
|
|||||||
visual,
|
visual,
|
||||||
"No IQDB matches right now.",
|
"No IQDB matches right now.",
|
||||||
)}
|
)}
|
||||||
{column("Auto-uploaded & Indexed", completed, "Nothing indexed yet.")}
|
{column(
|
||||||
|
"Auto-uploaded & Indexed",
|
||||||
|
completed,
|
||||||
|
"Nothing indexed yet.",
|
||||||
|
completed.length > 0 ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => dismissAll(completed)}
|
||||||
|
title="Dismiss every indexed record"
|
||||||
|
className="rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
||||||
|
>
|
||||||
|
dismiss all
|
||||||
|
</button>
|
||||||
|
) : null,
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{modalFor ? (
|
{modalFor ? (
|
||||||
|
|||||||
@@ -135,6 +135,7 @@ export interface TempUpload {
|
|||||||
iqdb_data: E621IqdbCandidate[] | null;
|
iqdb_data: E621IqdbCandidate[] | null;
|
||||||
library_j_id: string | null;
|
library_j_id: string | null;
|
||||||
file_url: string | null;
|
file_url: string | null;
|
||||||
|
preview_url: string | null;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user