diff --git a/ROADMAP.md b/ROADMAP.md index f9f2d2b..6acd347 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,6 +22,13 @@ they land. - [x] Status filter (matched / custom / unknown — not_found/deleted arrive with the e621 match cache) +- [x] **Ephemeral similarity check** (`/similar`) + - [x] Drop a file: exact MD5 match, perceptual matches against the library, + and e621 IQDB candidates (auto-run for images) + - [x] Nothing enters the library: temp files are wiped on startup, after + `SIMILARITY_TTL_MINUTES` (default 30), on demand and by + `manage.py cleanup_similarity` + ## 2. e621 integration - [x] **Download progress bar on the detail view** @@ -116,6 +123,8 @@ Files now stage first and are resolved before entering the library. - [ ] Guest blacklist refresh on a timer (`refresh_guest_blacklist` via cron/systemd) - [ ] Follow sync on a timer (`sync_followed_tags` + `sync_followed_pools` via cron/systemd, e.g. every 30 minutes) +- [ ] Similarity temp cleanup on a timer (`cleanup_similarity` via cron; the + TTL also cleans lazily when new checks are created) - [ ] Production setup: build the SPA, serve via Nginx (static + `/media` + `/library`), systemd unit for Waitress - [ ] Automated tests (backend API + frontend components) diff --git a/backend/.env.example b/backend/.env.example index b1e0fc7..1ee89e8 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -18,3 +18,7 @@ REDIS_URL=redis://127.0.0.1:6380/1 # Redis by `manage.py refresh_guest_blacklist`. # GUEST_BLACKLIST_FALLBACK=young,cub,shota,loli,child,underage # GUEST_BLACKLIST_TTL=3600 + +# Ephemeral similarity checks: temp files are wiped on startup and after +# this many minutes. +# SIMILARITY_TTL_MINUTES=30 diff --git a/backend/apps/library/apps.py b/backend/apps/library/apps.py index 1a1d68e..a498075 100644 --- a/backend/apps/library/apps.py +++ b/backend/apps/library/apps.py @@ -1,5 +1,21 @@ +import logging + from django.apps import AppConfig +logger = logging.getLogger(__name__) + class LibraryConfig(AppConfig): name = "apps.library" + + def ready(self): + """Ephemeral similarity-check files never survive a restart.""" + from . import similarity + + try: + deleted = similarity.purge_similarity_files() + except OSError as exc: + logger.warning("Similarity file purge skipped: %s", exc) + else: + if deleted: + logger.info("Purged %s similarity file(s) on startup", deleted) diff --git a/backend/apps/library/management/commands/cleanup_similarity.py b/backend/apps/library/management/commands/cleanup_similarity.py new file mode 100644 index 0000000..81e3a05 --- /dev/null +++ b/backend/apps/library/management/commands/cleanup_similarity.py @@ -0,0 +1,46 @@ +from datetime import timedelta + +from django.conf import settings +from django.core.management.base import BaseCommand +from django.utils import timezone + +from apps.library.similarity import purge_similarity_checks + + +class Command(BaseCommand): + help = ( + "Delete ephemeral similarity-check files. By default removes checks " + "older than SIMILARITY_TTL_MINUTES; use --all to wipe every one." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--minutes", + type=int, + default=None, + help="Delete checks older than N minutes " + "(default: the configured TTL, currently " + f"{getattr(settings, 'SIMILARITY_TTL_MINUTES', 30)}).", + ) + parser.add_argument( + "--all", + action="store_true", + help="Delete every similarity check regardless of age.", + ) + + def handle(self, *args, **options): + if options["all"]: + deleted = purge_similarity_checks() + scope = "all" + else: + minutes = ( + options["minutes"] + if options["minutes"] is not None + else getattr(settings, "SIMILARITY_TTL_MINUTES", 30) + ) + cutoff = timezone.now() - timedelta(minutes=minutes) + deleted = purge_similarity_checks(cutoff) + scope = f"older than {minutes} minute(s)" + self.stdout.write( + self.style.SUCCESS(f"Deleted {deleted} similarity check(s) ({scope}).") + ) diff --git a/backend/apps/library/migrations/0010_similaritycheck.py b/backend/apps/library/migrations/0010_similaritycheck.py new file mode 100644 index 0000000..8aff32f --- /dev/null +++ b/backend/apps/library/migrations/0010_similaritycheck.py @@ -0,0 +1,33 @@ +# Generated by Django 6.1.1 on 2026-09-17 23:18 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('library', '0009_mediaitem_e621_checked_at_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='SimilarityCheck', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('file', models.FileField(blank=True, upload_to='similarity/')), + ('original_filename', models.CharField(max_length=255)), + ('md5', models.CharField(blank=True, db_index=True, default='', max_length=32)), + ('size', models.BigIntegerField(default=0)), + ('results', models.JSONField(blank=True, default=dict)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='similarity_checks', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/backend/apps/library/models.py b/backend/apps/library/models.py index 6176997..02f2c4a 100644 --- a/backend/apps/library/models.py +++ b/backend/apps/library/models.py @@ -281,3 +281,32 @@ class MatchTask(models.Model): def __str__(self): return f"Match scan {self.id} ({self.status})" + + +class SimilarityCheck(models.Model): + """An ephemeral image uploaded only to check what it looks like. + + Never enters the library: files live in the media temp folder and are + purged on startup, by the TTL and by `manage.py cleanup_similarity`. + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="similarity_checks", + ) + file = models.FileField(upload_to="similarity/", blank=True) + original_filename = models.CharField(max_length=255) + md5 = models.CharField(max_length=32, db_index=True, blank=True, default="") + size = models.BigIntegerField(default=0) + results = models.JSONField(default=dict, blank=True) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self): + return f"Similarity check {self.id} ({self.original_filename})" diff --git a/backend/apps/library/serializers.py b/backend/apps/library/serializers.py index 60a2619..0dea0df 100644 --- a/backend/apps/library/serializers.py +++ b/backend/apps/library/serializers.py @@ -1,10 +1,19 @@ import os +from datetime import timedelta from pathlib import Path +from django.conf import settings from django.core import signing from rest_framework import serializers -from .models import MediaItem, MediaLocation, TempUpload, DownloadTask, MatchTask +from .models import ( + DownloadTask, + MatchTask, + MediaItem, + MediaLocation, + SimilarityCheck, + TempUpload, +) from .services import ( MEDIA_FILE_SALT, UPLOAD_FILE_SALT, @@ -246,3 +255,45 @@ class MatchTaskSerializer(serializers.ModelSerializer): "updated_at", ] read_only_fields = fields + + +class SimilarityCheckSerializer(serializers.ModelSerializer): + check_id = serializers.UUIDField(source="id", read_only=True) + file_url = serializers.SerializerMethodField() + expires_at = serializers.SerializerMethodField() + + class Meta: + model = SimilarityCheck + fields = [ + "check_id", + "original_filename", + "md5", + "size", + "file_url", + "results", + "expires_at", + "created_at", + ] + read_only_fields = fields + + def _request_user(self): + request = self.context.get("request") + user = getattr(request, "user", None) + if user is None or not getattr(user, "is_authenticated", False): + return None + return user + + def get_file_url(self, obj): + """Signed URL so tags can preview the temporary upload.""" + user = self._request_user() + if user is None or not obj.file: + return None + signature = signing.dumps( + {"check": str(obj.id), "user": user.id}, + salt=UPLOAD_FILE_SALT, + ) + return f"/api/similarity/{obj.id}/file/?sig={signature}" + + def get_expires_at(self, obj): + ttl = int(getattr(settings, "SIMILARITY_TTL_MINUTES", 30)) + return obj.created_at + timedelta(minutes=ttl) diff --git a/backend/apps/library/similarity.py b/backend/apps/library/similarity.py new file mode 100644 index 0000000..fc3ea72 --- /dev/null +++ b/backend/apps/library/similarity.py @@ -0,0 +1,159 @@ +"""Ephemeral similarity checks. + +A file lands in the media temp folder only long enough to answer "is this +already in the library, and what does it look like on e621?". Nothing is +indexed: files are deleted on startup, after the TTL, on request through the +API, and by `manage.py cleanup_similarity` (for cron). +""" + +import logging +from datetime import timedelta +from pathlib import Path + +from django.conf import settings +from django.core import signing +from django.utils import timezone +from rest_framework import mixins, status, viewsets +from rest_framework.decorators import action +from rest_framework.parsers import FormParser, JSONParser, MultiPartParser +from rest_framework.permissions import AllowAny, IsAuthenticated +from rest_framework.response import Response + +from . import services +from .models import MediaItem, SimilarityCheck +from .serializers import SimilarityCheckSerializer +from .tools import item_brief +from .uploads import find_library_matches + +logger = logging.getLogger(__name__) + +TTL_MINUTES = int(getattr(settings, "SIMILARITY_TTL_MINUTES", 30)) + + +def purge_similarity_files(): + """Delete every stored temp file without touching the database. + + Safe to call during app startup (``AppConfig.ready``) — database rows are + removed lazily by ``purge_expired`` and the cleanup command. + """ + root = Path(settings.MEDIA_ROOT) / "similarity" + if not root.exists(): + return 0 + deleted = 0 + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + try: + path.unlink() + deleted += 1 + except OSError as exc: # noqa: PERF203 - keep going past locked files + logger.warning("Could not delete %s: %s", path, exc) + for path in sorted(root.rglob("*"), reverse=True): + if path.is_dir(): + try: + path.rmdir() + except OSError: + pass + return deleted + + +def purge_similarity_checks(older_than=None): + """Delete checks (rows and files). ``older_than=None`` wipes them all.""" + queryset = SimilarityCheck.objects.all() + if older_than is not None: + queryset = queryset.filter(created_at__lt=older_than) + deleted = 0 + for check in queryset.iterator(): + if check.file: + check.file.delete(save=False) + check.delete() + deleted += 1 + return deleted + + +def purge_expired(): + """Remove checks past the TTL; called lazily before creating new ones.""" + return purge_similarity_checks(timezone.now() - timedelta(minutes=TTL_MINUTES)) + + +class SimilarityCheckViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): + """Check an upload against the library — exact MD5 plus visual matches.""" + + serializer_class = SimilarityCheckSerializer + permission_classes = [IsAuthenticated] + parser_classes = [MultiPartParser, FormParser, JSONParser] + http_method_names = ["get", "post", "delete", "head", "options"] + + def get_queryset(self): + queryset = SimilarityCheck.objects.all() + user = self.request.user + if not (user.is_staff or user.is_superuser): + queryset = queryset.filter(user=user) + return queryset + + def create(self, request): + upload = request.FILES.get("file") + if upload is None: + return Response( + {"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST + ) + extension = Path(upload.name).suffix.lower() + if extension not in services.ALLOWED_EXTENSIONS: + return Response( + {"detail": f"Unsupported file type: {extension or 'unknown'}"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + purge_expired() + check = SimilarityCheck.objects.create( + user=request.user, + file=upload, + original_filename=upload.name, + size=upload.size, + ) + check.md5 = services.compute_md5(check.file.path) + + exact = MediaItem.objects.filter(md5=check.md5).first() + matches = find_library_matches(check.file.path, limit=12, user=request.user) + if exact is not None: + exact_j_id = f"J-{exact.id}" + matches = [ + match for match in matches if match.get("j_id") != exact_j_id + ] + check.results = { + "exact": item_brief(exact, request) if exact is not None else None, + "matches": matches, + } + check.save(update_fields=["md5", "results"]) + return Response( + self.get_serializer(check).data, status=status.HTTP_201_CREATED + ) + + @action(detail=True, methods=["get", "head"], permission_classes=[AllowAny]) + def file(self, request, pk=None): + """Serve the temp file; accepts a signed URL like staged uploads.""" + check = None + if request.user.is_authenticated: + check = self.get_queryset().filter(pk=pk).first() + else: + signature = request.query_params.get("sig") + payload = None + if signature: + try: + payload = signing.loads( + signature, salt=services.UPLOAD_FILE_SALT, max_age=86400 + ) + except signing.BadSignature: + payload = None + if payload and payload.get("check") == str(pk): + check = SimilarityCheck.objects.filter(pk=pk).first() + if check is None or not check.file: + return Response( + {"detail": "Not found."}, status=status.HTTP_404_NOT_FOUND + ) + return services.serve_file(request, check.file.path) diff --git a/backend/apps/library/urls.py b/backend/apps/library/urls.py index d1a35a8..39541b1 100644 --- a/backend/apps/library/urls.py +++ b/backend/apps/library/urls.py @@ -10,6 +10,7 @@ from .tools import ( VisualGroupsView, VisualMatchesView, ) +from .similarity import SimilarityCheckViewSet from .uploads import TempUploadViewSet from .views import ( ClientDownloadView, @@ -23,6 +24,7 @@ router.register("files", MediaItemViewSet, basename="file") router.register("uploads", TempUploadViewSet, basename="upload") router.register("online/downloads", DownloadTaskViewSet, basename="download") router.register("matches", MatchTaskViewSet, basename="match") +router.register("similarity", SimilarityCheckViewSet, basename="similarity") urlpatterns = [ path("", include(router.urls)), diff --git a/backend/config/settings.py b/backend/config/settings.py index 9eb2cf0..3a1c68a 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -170,6 +170,10 @@ GUEST_BLACKLIST_TTL = int(os.getenv("GUEST_BLACKLIST_TTL", "3600")) # Similarity threshold for flagging staged uploads that match library items. VISUAL_MATCH_THRESHOLD = float(os.getenv("VISUAL_MATCH_THRESHOLD", "0.9")) +# Ephemeral similarity-check uploads are deleted after this many minutes +# (and always on startup). +SIMILARITY_TTL_MINUTES = int(os.getenv("SIMILARITY_TTL_MINUTES", "30")) + # Redis cache (run via docker compose at the repo root), shared by web # workers and management commands (e.g. the mirrored guest blacklist). CACHES = { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6d7d5a7..04abc0c 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -22,6 +22,7 @@ import Md5Redirect from "@/features/library/Md5Redirect"; import OnlinePage from "@/features/online/OnlinePage"; import PoolDetailPage from "@/features/pools/PoolDetailPage"; import PoolsPage from "@/features/pools/PoolsPage"; +import SimilarPage from "@/features/similar/SimilarPage"; import UploadPage from "@/features/upload/UploadPage"; import UsersPage from "@/features/users/UsersPage"; import { isAgeVerified, markAgeVerified } from "@/lib/age"; @@ -107,6 +108,14 @@ export default function App() { } /> + + + + } + /> = { + s: "bg-ctp-green text-ctp-crust", + q: "bg-ctp-peach text-ctp-crust", + e: "bg-ctp-red text-ctp-crust", +}; + +export default function SimilarPage() { + const queryClient = useQueryClient(); + const credentials = useE621((state) => state.credentials); + const inputRef = useRef(null); + + const [file, setFile] = useState(null); + const [check, setCheck] = useState(null); + const [uploading, setUploading] = useState(false); + const [percent, setPercent] = useState(0); + const [error, setError] = useState(null); + const [dragging, setDragging] = useState(false); + + const [iqdb, setIqdb] = useState(null); + const [iqdbBusy, setIqdbBusy] = useState(false); + const [iqdbError, setIqdbError] = useState(null); + const [selected, setSelected] = useState(null); + + const configured = Boolean(credentials?.configured); + + const deleteMutation = useMutation({ + mutationFn: () => + api(`/api/similarity/${check?.check_id}/`, { method: "DELETE" }), + onSuccess: () => { + setCheck(null); + setFile(null); + setIqdb(null); + setSelected(null); + setError(null); + setIqdbError(null); + if (inputRef.current) inputRef.current.value = ""; + }, + }); + + async function startCheck(selectedFile: File) { + setFile(selectedFile); + setCheck(null); + setIqdb(null); + setSelected(null); + setError(null); + setIqdbError(null); + setPercent(0); + setUploading(true); + try { + const data = await uploadFile( + selectedFile, + {}, + setPercent, + "/api/similarity/", + ); + setCheck(data); + void queryClient.invalidateQueries({ queryKey: ["files"] }); + } catch (err) { + setError(errorMessage(err)); + } finally { + setUploading(false); + } + + if (VIDEO_RE.test(selectedFile.name)) return; + if (!configured) return; + setIqdbBusy(true); + try { + const results = await iqdbSearch( + effectiveCredentials(credentials), + selectedFile, + selectedFile.name, + ); + setIqdb(results); + } catch (err) { + setIqdbError(e621ErrorMessage(err)); + } finally { + setIqdbBusy(false); + } + } + + function handleInput(event: ChangeEvent) { + const chosen = event.target.files?.[0]; + if (chosen) void startCheck(chosen); + } + + function handleDrop(event: DragEvent) { + event.preventDefault(); + setDragging(false); + const dropped = event.dataTransfer.files?.[0]; + if (dropped) void startCheck(dropped); + } + + const isVideo = file ? VIDEO_RE.test(file.name) : false; + const exact = check?.results.exact ?? null; + const matches = check?.results.matches ?? []; + + return ( +
+
+

Similarity check

+

+ Drop a file to see whether it is already in the library and what it + looks like on e621. Nothing is added to the library — the temporary + copy is deleted automatically after a while and on the next server + start. +

+
+ +
{ + event.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={handleDrop} + className={cn( + "flex flex-col items-center justify-center gap-2 rounded-lg border border-dashed px-6 py-10 text-center transition", + dragging + ? "border-ctp-mauve bg-ctp-mauve/10" + : "border-ctp-surface1 bg-ctp-base", + )} + > + +

+ Drag a file here, or{" "} + +

+

+ Images get the full check (MD5, visual hashes, e621 IQDB); videos + only get the exact MD5 match. +

+ +
+ + {uploading ? ( +
+
+
+
+ + uploading… {percent}% + +
+ ) : null} + + {error ?

{error}

: null} + + {check ? ( +
+ + +
+
+

+ In the library (exact MD5) +

+ {exact ? ( + + {exact.thumbnail_url ? ( + {exact.j_id} + ) : null} + + + {exact.j_id} + + + {exact.filename} · {formatBytes(exact.size)} ·{" "} + {exact.location_count} cop + {exact.location_count === 1 ? "y" : "ies"} + {exact.e621_post_id + ? ` · e621 #${exact.e621_post_id}` + : ""} + + + + ) : ( +

+ Not in the library — no item has this MD5. +

+ )} +
+ +
+

+ Visually similar in the library{" "} + + {matches.length} + +

+ {isVideo ? ( +

+ Visual hashing works on images only. +

+ ) : matches.length === 0 ? ( +

+ Nothing above the similarity threshold. +

+ ) : ( +
+ {matches.map((match) => ( + + + {match.thumbnail_url ? ( + {match.j_id} + ) : null} + + {match.similarity}% + + + + {match.j_id} + + + ))} +
+ )} +
+ +
+
+

+ e621 IQDB +

+ {iqdbBusy ? ( + + searching… + + ) : iqdb ? ( + + {iqdb.length} candidate(s) + + ) : null} +
+ + {isVideo ? ( +

+ IQDB works on images only. +

+ ) : !configured ? ( +

+ + Add e621 credentials + {" "} + to reverse-search this image on e621. +

+ ) : iqdbError ? ( +

{iqdbError}

+ ) : iqdbBusy && !iqdb ? ( +
+ +
+ ) : iqdb && iqdb.length === 0 ? ( +

+ No similar posts found on e621. +

+ ) : iqdb ? ( + <> +
+ {iqdb.map((candidate) => { + const isExact = + candidate.md5 !== null && + check.md5 === candidate.md5; + return ( + + ); + })} +
+ + {selected ? ( +
+

+ Post #{selected.post_id} +

+

+ {selected.rating + ? (RATING_LABELS[selected.rating] ?? + selected.rating) + : "Unknown rating"} + {selected.score_total !== null + ? ` · ▲ ${selected.score_total}` + : ""} + {selected.fav_count !== null + ? ` · ${selected.fav_count} favs` + : ""} + {selected.width && selected.height + ? ` · ${selected.width}×${selected.height}` + : ""} +

+ {selected.tags_preview.length > 0 ? ( +
+ {selected.tags_preview.map((tag) => ( + + {tag} + + ))} +
+ ) : null} +
+ + View post + +
+
+ ) : ( +

+ Select a candidate for its rating, score, favorites and + tags. +

+ )} + + ) : ( +

+ Waiting for the library check… +

+ )} +
+
+
+ ) : null} +
+ ); +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index fc7b516..9b3f1a6 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -85,11 +85,12 @@ export async function api(path: string, options: ApiOptions = {}): Promise return (await response.json()) as T; } -export function uploadFile( +export function uploadFile( file: File, fields: Record = {}, onProgress?: (percent: number) => void, -): Promise { + path = "/api/uploads/", +): Promise { return new Promise((resolve, reject) => { const form = new FormData(); form.append("file", file); @@ -98,7 +99,7 @@ export function uploadFile( } const request = new XMLHttpRequest(); - request.open("POST", "/api/uploads/"); + request.open("POST", path); const token = getToken(); if (token) request.setRequestHeader("Authorization", `Token ${token}`); request.upload.addEventListener("progress", (event) => { @@ -114,7 +115,7 @@ export function uploadFile( data = null; } if (request.status >= 200 && request.status < 300) { - resolve(data as TempUpload); + resolve(data as T); } else { reject(new ApiError(request.status, data)); } diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 31f562a..89ddef2 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -265,6 +265,20 @@ export interface FollowFeed { results: FollowFeedEntry[]; } +export interface SimilarityCheck { + check_id: string; + original_filename: string; + md5: string; + size: number; + file_url: string | null; + results: { + exact: DuplicateEntry | null; + matches: DuplicateEntry[]; + }; + expires_at: string; + created_at: string; +} + export interface FollowCloud { status: "ready" | "building"; stale: boolean;