Add an ephemeral similarity check page
- /similar (nav: Similar): drop a file to get the exact MD5 match, the perceptual matches against the library, and e621 IQDB candidates (auto-run for images when credentials are configured). Read-only — nothing enters the library. - SimilarityCheck model + /api/similarity/ (create/list/retrieve/delete) with signed preview URLs and an expires_at timestamp. - Temp files are wiped on startup (AppConfig.ready, file-only so no database access during initialization), lazily past SIMILARITY_TTL_MINUTES (default 30, env-overridable), on delete, and by manage.py cleanup_similarity. - uploadFile() takes a target path; .env.example documents the TTL.
This commit is contained in:
@@ -18,3 +18,7 @@ REDIS_URL=redis://127.0.0.1:6380/1
|
||||
# Redis by `manage.py refresh_guest_blacklist`.
|
||||
# GUEST_BLACKLIST_FALLBACK=young,cub,shota,loli,child,underage
|
||||
# GUEST_BLACKLIST_TTL=3600
|
||||
|
||||
# Ephemeral similarity checks: temp files are wiped on startup and after
|
||||
# this many minutes.
|
||||
# SIMILARITY_TTL_MINUTES=30
|
||||
|
||||
@@ -1,5 +1,21 @@
|
||||
import logging
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LibraryConfig(AppConfig):
|
||||
name = "apps.library"
|
||||
|
||||
def ready(self):
|
||||
"""Ephemeral similarity-check files never survive a restart."""
|
||||
from . import similarity
|
||||
|
||||
try:
|
||||
deleted = similarity.purge_similarity_files()
|
||||
except OSError as exc:
|
||||
logger.warning("Similarity file purge skipped: %s", exc)
|
||||
else:
|
||||
if deleted:
|
||||
logger.info("Purged %s similarity file(s) on startup", deleted)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from datetime import timedelta
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.library.similarity import purge_similarity_checks
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Delete ephemeral similarity-check files. By default removes checks "
|
||||
"older than SIMILARITY_TTL_MINUTES; use --all to wipe every one."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--minutes",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Delete checks older than N minutes "
|
||||
"(default: the configured TTL, currently "
|
||||
f"{getattr(settings, 'SIMILARITY_TTL_MINUTES', 30)}).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Delete every similarity check regardless of age.",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if options["all"]:
|
||||
deleted = purge_similarity_checks()
|
||||
scope = "all"
|
||||
else:
|
||||
minutes = (
|
||||
options["minutes"]
|
||||
if options["minutes"] is not None
|
||||
else getattr(settings, "SIMILARITY_TTL_MINUTES", 30)
|
||||
)
|
||||
cutoff = timezone.now() - timedelta(minutes=minutes)
|
||||
deleted = purge_similarity_checks(cutoff)
|
||||
scope = f"older than {minutes} minute(s)"
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(f"Deleted {deleted} similarity check(s) ({scope}).")
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
# Generated by Django 6.1.1 on 2026-09-17 23:18
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('library', '0009_mediaitem_e621_checked_at_and_more'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='SimilarityCheck',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('file', models.FileField(blank=True, upload_to='similarity/')),
|
||||
('original_filename', models.CharField(max_length=255)),
|
||||
('md5', models.CharField(blank=True, db_index=True, default='', max_length=32)),
|
||||
('size', models.BigIntegerField(default=0)),
|
||||
('results', models.JSONField(blank=True, default=dict)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='similarity_checks', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -281,3 +281,32 @@ class MatchTask(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return f"Match scan {self.id} ({self.status})"
|
||||
|
||||
|
||||
class SimilarityCheck(models.Model):
|
||||
"""An ephemeral image uploaded only to check what it looks like.
|
||||
|
||||
Never enters the library: files live in the media temp folder and are
|
||||
purged on startup, by the TTL and by `manage.py cleanup_similarity`.
|
||||
"""
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="similarity_checks",
|
||||
)
|
||||
file = models.FileField(upload_to="similarity/", blank=True)
|
||||
original_filename = models.CharField(max_length=255)
|
||||
md5 = models.CharField(max_length=32, db_index=True, blank=True, default="")
|
||||
size = models.BigIntegerField(default=0)
|
||||
results = models.JSONField(default=dict, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
|
||||
def __str__(self):
|
||||
return f"Similarity check {self.id} ({self.original_filename})"
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core import signing
|
||||
from rest_framework import serializers
|
||||
|
||||
from .models import MediaItem, MediaLocation, TempUpload, DownloadTask, MatchTask
|
||||
from .models import (
|
||||
DownloadTask,
|
||||
MatchTask,
|
||||
MediaItem,
|
||||
MediaLocation,
|
||||
SimilarityCheck,
|
||||
TempUpload,
|
||||
)
|
||||
from .services import (
|
||||
MEDIA_FILE_SALT,
|
||||
UPLOAD_FILE_SALT,
|
||||
@@ -246,3 +255,45 @@ class MatchTaskSerializer(serializers.ModelSerializer):
|
||||
"updated_at",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
|
||||
class SimilarityCheckSerializer(serializers.ModelSerializer):
|
||||
check_id = serializers.UUIDField(source="id", read_only=True)
|
||||
file_url = serializers.SerializerMethodField()
|
||||
expires_at = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = SimilarityCheck
|
||||
fields = [
|
||||
"check_id",
|
||||
"original_filename",
|
||||
"md5",
|
||||
"size",
|
||||
"file_url",
|
||||
"results",
|
||||
"expires_at",
|
||||
"created_at",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
def _request_user(self):
|
||||
request = self.context.get("request")
|
||||
user = getattr(request, "user", None)
|
||||
if user is None or not getattr(user, "is_authenticated", False):
|
||||
return None
|
||||
return user
|
||||
|
||||
def get_file_url(self, obj):
|
||||
"""Signed URL so <img> tags can preview the temporary upload."""
|
||||
user = self._request_user()
|
||||
if user is None or not obj.file:
|
||||
return None
|
||||
signature = signing.dumps(
|
||||
{"check": str(obj.id), "user": user.id},
|
||||
salt=UPLOAD_FILE_SALT,
|
||||
)
|
||||
return f"/api/similarity/{obj.id}/file/?sig={signature}"
|
||||
|
||||
def get_expires_at(self, obj):
|
||||
ttl = int(getattr(settings, "SIMILARITY_TTL_MINUTES", 30))
|
||||
return obj.created_at + timedelta(minutes=ttl)
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Ephemeral similarity checks.
|
||||
|
||||
A file lands in the media temp folder only long enough to answer "is this
|
||||
already in the library, and what does it look like on e621?". Nothing is
|
||||
indexed: files are deleted on startup, after the TTL, on request through the
|
||||
API, and by `manage.py cleanup_similarity` (for cron).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from django.conf import settings
|
||||
from django.core import signing
|
||||
from django.utils import timezone
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticated
|
||||
from rest_framework.response import Response
|
||||
|
||||
from . import services
|
||||
from .models import MediaItem, SimilarityCheck
|
||||
from .serializers import SimilarityCheckSerializer
|
||||
from .tools import item_brief
|
||||
from .uploads import find_library_matches
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TTL_MINUTES = int(getattr(settings, "SIMILARITY_TTL_MINUTES", 30))
|
||||
|
||||
|
||||
def purge_similarity_files():
|
||||
"""Delete every stored temp file without touching the database.
|
||||
|
||||
Safe to call during app startup (``AppConfig.ready``) — database rows are
|
||||
removed lazily by ``purge_expired`` and the cleanup command.
|
||||
"""
|
||||
root = Path(settings.MEDIA_ROOT) / "similarity"
|
||||
if not root.exists():
|
||||
return 0
|
||||
deleted = 0
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
path.unlink()
|
||||
deleted += 1
|
||||
except OSError as exc: # noqa: PERF203 - keep going past locked files
|
||||
logger.warning("Could not delete %s: %s", path, exc)
|
||||
for path in sorted(root.rglob("*"), reverse=True):
|
||||
if path.is_dir():
|
||||
try:
|
||||
path.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
return deleted
|
||||
|
||||
|
||||
def purge_similarity_checks(older_than=None):
|
||||
"""Delete checks (rows and files). ``older_than=None`` wipes them all."""
|
||||
queryset = SimilarityCheck.objects.all()
|
||||
if older_than is not None:
|
||||
queryset = queryset.filter(created_at__lt=older_than)
|
||||
deleted = 0
|
||||
for check in queryset.iterator():
|
||||
if check.file:
|
||||
check.file.delete(save=False)
|
||||
check.delete()
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
def purge_expired():
|
||||
"""Remove checks past the TTL; called lazily before creating new ones."""
|
||||
return purge_similarity_checks(timezone.now() - timedelta(minutes=TTL_MINUTES))
|
||||
|
||||
|
||||
class SimilarityCheckViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.RetrieveModelMixin,
|
||||
mixins.DestroyModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
"""Check an upload against the library — exact MD5 plus visual matches."""
|
||||
|
||||
serializer_class = SimilarityCheckSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
parser_classes = [MultiPartParser, FormParser, JSONParser]
|
||||
http_method_names = ["get", "post", "delete", "head", "options"]
|
||||
|
||||
def get_queryset(self):
|
||||
queryset = SimilarityCheck.objects.all()
|
||||
user = self.request.user
|
||||
if not (user.is_staff or user.is_superuser):
|
||||
queryset = queryset.filter(user=user)
|
||||
return queryset
|
||||
|
||||
def create(self, request):
|
||||
upload = request.FILES.get("file")
|
||||
if upload is None:
|
||||
return Response(
|
||||
{"detail": "A file is required."}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
extension = Path(upload.name).suffix.lower()
|
||||
if extension not in services.ALLOWED_EXTENSIONS:
|
||||
return Response(
|
||||
{"detail": f"Unsupported file type: {extension or 'unknown'}"},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
purge_expired()
|
||||
check = SimilarityCheck.objects.create(
|
||||
user=request.user,
|
||||
file=upload,
|
||||
original_filename=upload.name,
|
||||
size=upload.size,
|
||||
)
|
||||
check.md5 = services.compute_md5(check.file.path)
|
||||
|
||||
exact = MediaItem.objects.filter(md5=check.md5).first()
|
||||
matches = find_library_matches(check.file.path, limit=12, user=request.user)
|
||||
if exact is not None:
|
||||
exact_j_id = f"J-{exact.id}"
|
||||
matches = [
|
||||
match for match in matches if match.get("j_id") != exact_j_id
|
||||
]
|
||||
check.results = {
|
||||
"exact": item_brief(exact, request) if exact is not None else None,
|
||||
"matches": matches,
|
||||
}
|
||||
check.save(update_fields=["md5", "results"])
|
||||
return Response(
|
||||
self.get_serializer(check).data, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["get", "head"], permission_classes=[AllowAny])
|
||||
def file(self, request, pk=None):
|
||||
"""Serve the temp file; accepts a signed URL like staged uploads."""
|
||||
check = None
|
||||
if request.user.is_authenticated:
|
||||
check = self.get_queryset().filter(pk=pk).first()
|
||||
else:
|
||||
signature = request.query_params.get("sig")
|
||||
payload = None
|
||||
if signature:
|
||||
try:
|
||||
payload = signing.loads(
|
||||
signature, salt=services.UPLOAD_FILE_SALT, max_age=86400
|
||||
)
|
||||
except signing.BadSignature:
|
||||
payload = None
|
||||
if payload and payload.get("check") == str(pk):
|
||||
check = SimilarityCheck.objects.filter(pk=pk).first()
|
||||
if check is None or not check.file:
|
||||
return Response(
|
||||
{"detail": "Not found."}, status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
return services.serve_file(request, check.file.path)
|
||||
@@ -10,6 +10,7 @@ from .tools import (
|
||||
VisualGroupsView,
|
||||
VisualMatchesView,
|
||||
)
|
||||
from .similarity import SimilarityCheckViewSet
|
||||
from .uploads import TempUploadViewSet
|
||||
from .views import (
|
||||
ClientDownloadView,
|
||||
@@ -23,6 +24,7 @@ router.register("files", MediaItemViewSet, basename="file")
|
||||
router.register("uploads", TempUploadViewSet, basename="upload")
|
||||
router.register("online/downloads", DownloadTaskViewSet, basename="download")
|
||||
router.register("matches", MatchTaskViewSet, basename="match")
|
||||
router.register("similarity", SimilarityCheckViewSet, basename="similarity")
|
||||
|
||||
urlpatterns = [
|
||||
path("", include(router.urls)),
|
||||
|
||||
@@ -170,6 +170,10 @@ GUEST_BLACKLIST_TTL = int(os.getenv("GUEST_BLACKLIST_TTL", "3600"))
|
||||
# Similarity threshold for flagging staged uploads that match library items.
|
||||
VISUAL_MATCH_THRESHOLD = float(os.getenv("VISUAL_MATCH_THRESHOLD", "0.9"))
|
||||
|
||||
# Ephemeral similarity-check uploads are deleted after this many minutes
|
||||
# (and always on startup).
|
||||
SIMILARITY_TTL_MINUTES = int(os.getenv("SIMILARITY_TTL_MINUTES", "30"))
|
||||
|
||||
# Redis cache (run via docker compose at the repo root), shared by web
|
||||
# workers and management commands (e.g. the mirrored guest blacklist).
|
||||
CACHES = {
|
||||
|
||||
Reference in New Issue
Block a user