diff --git a/ROADMAP.md b/ROADMAP.md index 0b07e8e..7aa0033 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,15 +5,16 @@ they land. ## 1. Library management -- [ ] **Duplicates engine** - - [ ] Exact MD5 duplicate detection with grouped results - - [ ] Perceptual similarity (aHash, dHash, pHash, wHash) with threshold slider - and algorithm toggles (needs `imagehash`/`imgdd` server-side) - - [ ] Visual similarity groups: pagination, "keep one / delete rest", dismiss group -- [ ] **Delete & storage page** - - [ ] Storage overview (watched folder, media folder) - - [ ] Delete by J-ID with preview grid and bulk selection - - [ ] Temp folder cleanup +- [x] **Duplicates engine** + - [x] Exact MD5 duplicate detection with grouped results + - [x] Perceptual similarity (aHash, dHash, pHash, wHash) with threshold slider + and algorithm toggles (`imagehash` server-side) + - [x] Visual similarity groups: pagination, selection, delete and dismiss +- [x] **Delete & storage page** + - [x] Storage overview (watched folder, media folder, temp) + - [x] Delete by J-ID with preview grid and bulk selection (plus per-copy + deletion of duplicate locations) + - [x] Temp folder cleanup - [ ] **Library search upgrades** - [ ] Search by tags (custom + e621 tags), not just filename - [ ] Tag cloud from the library @@ -58,8 +59,8 @@ Files now stage first and are resolved before entering the library. - [x] **IQDB similarity on upload** (SPA-driven) - [x] Automatic + manual IQDB checks with candidate posts - [x] "Visual Similarity Detected" state with candidate picker - - [ ] Perceptual-hash comparison against the library (shared with the - duplicates engine, still pending) + - [x] Perceptual-hash comparison against the library (staged uploads are + flagged with their library matches as soon as they land) - [x] **Upload UI** - [x] Three-column board: Pending & Unmatched / Visual Similarity Detected / Auto-uploaded & Indexed @@ -80,7 +81,8 @@ Files now stage first and are resolved before entering the library. - [ ] Toasts instead of inline messages / confirm dialogs - [ ] Mobile drawer polish for metadata panels (design spec §layout) -- [ ] Avatar picker (choose from library) +- [x] Profile pictures: staff Users page sets avatars from library J-IDs + (self-service picker in Account still pending) - [ ] Profile extras (per-user browse preferences) - [ ] Command palette: tag finder, recent searches diff --git a/backend/apps/accounts/api_urls.py b/backend/apps/accounts/api_urls.py new file mode 100644 index 0000000..18b93dd --- /dev/null +++ b/backend/apps/accounts/api_urls.py @@ -0,0 +1,8 @@ +from rest_framework.routers import DefaultRouter + +from .views import UserViewSet + +router = DefaultRouter() +router.register("users", UserViewSet, basename="user") + +urlpatterns = router.urls diff --git a/backend/apps/accounts/migrations/0004_user_avatar.py b/backend/apps/accounts/migrations/0004_user_avatar.py new file mode 100644 index 0000000..86fe4ee --- /dev/null +++ b/backend/apps/accounts/migrations/0004_user_avatar.py @@ -0,0 +1,20 @@ +# Generated by Django 6.1.1 on 2026-09-17 17:34 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0003_user_role'), + ('library', '0006_mediaitem_ahash_mediaitem_dhash_mediaitem_phash_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='avatar', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='library.mediaitem'), + ), + ] diff --git a/backend/apps/accounts/models.py b/backend/apps/accounts/models.py index 3599de9..b27269e 100644 --- a/backend/apps/accounts/models.py +++ b/backend/apps/accounts/models.py @@ -15,6 +15,13 @@ class User(AbstractUser): ] role = models.CharField(max_length=20, choices=ROLE_CHOICES, default=ROLE_USER) + avatar = models.ForeignKey( + "library.MediaItem", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="+", + ) e621_username = models.CharField(max_length=100, blank=True, default="") e621_api_key = models.CharField(max_length=100, blank=True, default="") e621_base_url = models.CharField(max_length=200, default="https://e621.net") diff --git a/backend/apps/accounts/serializers.py b/backend/apps/accounts/serializers.py index 89e30b4..bb0d7b2 100644 --- a/backend/apps/accounts/serializers.py +++ b/backend/apps/accounts/serializers.py @@ -1,12 +1,39 @@ +from pathlib import Path + from django.contrib.auth.password_validation import validate_password +from django.core import signing from rest_framework import serializers +from apps.library.services import MEDIA_FILE_SALT, VIDEO_EXTENSIONS + from .models import User +def signed_media_url(request, item): + """A signed media URL an tag can load for an authenticated user.""" + if item is None: + return None + user = getattr(request, "user", None) + if user is None or not getattr(user, "is_authenticated", False): + return None + location = item.locations.first() + is_video = ( + location is not None + and Path(location.rel_path).suffix.lower() in VIDEO_EXTENSIONS + ) + action = "thumbnail" if is_video else "raw" + signature = signing.dumps( + {"item": item.id, "user": user.id, "action": action}, + salt=MEDIA_FILE_SALT, + ) + return f"/api/files/J-{item.id}/{action}/?sig={signature}" + + class UserSerializer(serializers.ModelSerializer): e621_configured = serializers.BooleanField(read_only=True) can_upload = serializers.BooleanField(read_only=True) + avatar_j_id = serializers.SerializerMethodField() + avatar_url = serializers.SerializerMethodField() class Meta: model = User @@ -19,11 +46,54 @@ class UserSerializer(serializers.ModelSerializer): "date_joined", "role", "can_upload", + "avatar_j_id", + "avatar_url", "e621_username", "e621_configured", ] read_only_fields = fields + def get_avatar_j_id(self, obj): + return f"J-{obj.avatar_id}" if obj.avatar_id else None + + def get_avatar_url(self, obj): + return signed_media_url(self.context.get("request"), obj.avatar) + + +class UserListSerializer(serializers.ModelSerializer): + avatar_j_id = serializers.SerializerMethodField() + avatar_url = serializers.SerializerMethodField() + uploads_count = serializers.IntegerField(read_only=True) + + class Meta: + model = User + fields = [ + "id", + "username", + "email", + "role", + "is_staff", + "is_superuser", + "date_joined", + "avatar_j_id", + "avatar_url", + "uploads_count", + ] + read_only_fields = fields + + def get_avatar_j_id(self, obj): + return f"J-{obj.avatar_id}" if obj.avatar_id else None + + def get_avatar_url(self, obj): + return signed_media_url(self.context.get("request"), obj.avatar) + + +class UserUpdateSerializer(serializers.Serializer): + avatar_j_id = serializers.CharField( + required=False, allow_blank=True, allow_null=True + ) + role = serializers.ChoiceField(choices=User.ROLE_CHOICES, required=False) + class RegisterSerializer(serializers.ModelSerializer): password = serializers.CharField(write_only=True, validators=[validate_password]) diff --git a/backend/apps/accounts/views.py b/backend/apps/accounts/views.py index ad92d31..d65fed2 100644 --- a/backend/apps/accounts/views.py +++ b/backend/apps/accounts/views.py @@ -1,13 +1,19 @@ -from rest_framework import status +from rest_framework import mixins, permissions, status, viewsets from rest_framework.authtoken.models import Token from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +from django.db.models import Count, Q +from apps.library.models import MediaItem + +from .models import User from .serializers import ( E621CredentialsSerializer, RegisterSerializer, + UserListSerializer, UserSerializer, + UserUpdateSerializer, ) @@ -20,7 +26,10 @@ class RegisterView(APIView): user = serializer.save() token, _ = Token.objects.get_or_create(user=user) return Response( - {"token": token.key, "user": UserSerializer(user).data}, + { + "token": token.key, + "user": UserSerializer(user, context={"request": request}).data, + }, status=status.HTTP_201_CREATED, ) @@ -29,7 +38,7 @@ class MeView(APIView): permission_classes = [IsAuthenticated] def get(self, request): - return Response(UserSerializer(request.user).data) + return Response(UserSerializer(request.user, context={"request": request}).data) class LogoutView(APIView): @@ -71,3 +80,81 @@ class E621CredentialsView(APIView): update_fields=["e621_username", "e621_api_key", "e621_base_url"] ) return Response(self._payload(user)) + + +class IsStaffUser(permissions.BasePermission): + message = "Staff only." + + def has_permission(self, request, view): + user = request.user + return bool( + user + and user.is_authenticated + and ( + user.is_superuser + or user.is_staff + or user.role == user.ROLE_STAFF + ) + ) + + +class UserViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Staff user directory: roles and J-ID avatars.""" + + permission_classes = [IsStaffUser] + http_method_names = ["get", "patch", "head", "options"] + + def get_queryset(self): + queryset = User.objects.annotate(uploads_count=Count("uploads")).order_by("id") + search = self.request.query_params.get("search", "").strip() + if search: + queryset = queryset.filter( + Q(username__icontains=search) | Q(email__icontains=search) + ) + return queryset + + def get_serializer_class(self): + if self.request.method in {"PATCH", "PUT"}: + return UserUpdateSerializer + return UserListSerializer + + def update(self, request, *args, **kwargs): + user = self.get_object() + serializer = UserUpdateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + data = serializer.validated_data + + update_fields = [] + if "role" in data: + user.role = data["role"] + update_fields.append("role") + if "avatar_j_id" in data: + value = str(data.get("avatar_j_id") or "").strip() + if not value: + user.avatar = None + else: + numeric = value[2:] if value.upper().startswith("J-") else value + item = ( + MediaItem.objects.filter(pk=int(numeric)).first() + if numeric.isdigit() + else None + ) + if item is None: + return Response( + {"detail": f"No library item {value}."}, + status=status.HTTP_400_BAD_REQUEST, + ) + user.avatar = item + update_fields.append("avatar") + if update_fields: + user.save(update_fields=update_fields) + + return Response( + UserListSerializer(user, context=self.get_serializer_context()).data + ) + + partial_update = update diff --git a/backend/apps/library/downloads.py b/backend/apps/library/downloads.py index e0eac27..ac1c6f9 100644 --- a/backend/apps/library/downloads.py +++ b/backend/apps/library/downloads.py @@ -3,10 +3,12 @@ import logging import threading import time +from datetime import timedelta from pathlib import Path from urllib.parse import urlparse from django.conf import settings +from django.db import connection from django.utils import timezone from . import services @@ -14,6 +16,27 @@ from .models import DownloadTask logger = logging.getLogger(__name__) +STALE_AFTER = timedelta(minutes=30) + + +def reap_stale_downloads(): + """Mark tasks left hanging by a recycled worker as failed. + + Gunicorn recycles workers (--max-requests, timeouts); a download thread + dies with its worker, so long-stuck tasks are surfaced as errors instead + of pretending to run forever. + """ + cutoff = timezone.now() - STALE_AFTER + return DownloadTask.objects.filter( + status__in=[DownloadTask.STATUS_PENDING, DownloadTask.STATUS_DOWNLOADING], + updated_at__lt=cutoff, + ).update( + status=DownloadTask.STATUS_ERROR, + error="The worker restarted before this download finished.", + speed=None, + updated_at=timezone.now(), + ) + def start_download_task(task_id): thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True) @@ -79,6 +102,7 @@ def run_download_task(task_id): ) item, _, location, _ = services.index_file(destination, folder) services.rename_location_to_j_id(item, location) + services.ensure_visual_hashes(item) update_fields = [] if item.uploaded_by_id is None and task.user_id is not None: @@ -120,3 +144,7 @@ def run_download_task(task_id): speed=None, updated_at=timezone.now(), ) + finally: + # Background threads hold their own DB connection; release it so + # Gunicorn workers do not leak connections when threads finish. + connection.close() diff --git a/backend/apps/library/management/commands/compute_visual_hashes.py b/backend/apps/library/management/commands/compute_visual_hashes.py new file mode 100644 index 0000000..a26b989 --- /dev/null +++ b/backend/apps/library/management/commands/compute_visual_hashes.py @@ -0,0 +1,43 @@ +from django.core.management.base import BaseCommand +from django.db.models import Q + +from apps.library.models import MediaItem +from apps.library.services import HASH_FIELDS, ensure_visual_hashes + + +class Command(BaseCommand): + help = "Compute perceptual hashes for library items missing them." + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Recompute hashes even when they already exist.", + ) + + def handle(self, *args, **options): + queryset = MediaItem.objects.prefetch_related("locations").order_by("id") + if not options["force"]: + missing = Q() + for field in HASH_FIELDS: + missing |= Q(**{field: ""}) + queryset = queryset.filter(missing) + + total = queryset.count() + if total == 0: + self.stdout.write(self.style.SUCCESS("All items already have hashes.")) + return + + self.stdout.write(f"Computing hashes for {total} item(s) ...") + done = 0 + for item in queryset.iterator(chunk_size=100): + if options["force"]: + item.ahash = item.dhash = item.phash = item.whash = "" + ensure_visual_hashes(item) + done += 1 + if done % 100 == 0: + self.stdout.write(f"Processed {done} ...") + + self.stdout.write( + self.style.SUCCESS(f"Done. Processed {done} item(s).") + ) diff --git a/backend/apps/library/migrations/0006_mediaitem_ahash_mediaitem_dhash_mediaitem_phash_and_more.py b/backend/apps/library/migrations/0006_mediaitem_ahash_mediaitem_dhash_mediaitem_phash_and_more.py new file mode 100644 index 0000000..9a0db81 --- /dev/null +++ b/backend/apps/library/migrations/0006_mediaitem_ahash_mediaitem_dhash_mediaitem_phash_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 6.1.1 on 2026-09-17 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('library', '0005_downloadtask'), + ] + + operations = [ + migrations.AddField( + model_name='mediaitem', + name='ahash', + field=models.CharField(blank=True, db_index=True, default='', max_length=32), + ), + migrations.AddField( + model_name='mediaitem', + name='dhash', + field=models.CharField(blank=True, db_index=True, default='', max_length=32), + ), + migrations.AddField( + model_name='mediaitem', + name='phash', + field=models.CharField(blank=True, db_index=True, default='', max_length=32), + ), + migrations.AddField( + model_name='mediaitem', + name='whash', + field=models.CharField(blank=True, db_index=True, default='', max_length=32), + ), + ] diff --git a/backend/apps/library/migrations/0007_tempupload_visual_matches.py b/backend/apps/library/migrations/0007_tempupload_visual_matches.py new file mode 100644 index 0000000..648aa98 --- /dev/null +++ b/backend/apps/library/migrations/0007_tempupload_visual_matches.py @@ -0,0 +1,18 @@ +# Generated by Django 6.1.1 on 2026-09-17 17:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('library', '0006_mediaitem_ahash_mediaitem_dhash_mediaitem_phash_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='tempupload', + name='visual_matches', + field=models.JSONField(blank=True, null=True), + ), + ] diff --git a/backend/apps/library/models.py b/backend/apps/library/models.py index cd80e4c..45d56a6 100644 --- a/backend/apps/library/models.py +++ b/backend/apps/library/models.py @@ -24,6 +24,11 @@ class MediaItem(models.Model): hidden_from_guests = models.BooleanField(default=False, db_index=True) e621_post_id = models.IntegerField(null=True, blank=True, db_index=True) e621_data = models.JSONField(null=True, blank=True) + # Perceptual hashes (hex strings) used by the duplicates engine. + ahash = models.CharField(max_length=32, blank=True, default="", db_index=True) + dhash = models.CharField(max_length=32, blank=True, default="", db_index=True) + phash = models.CharField(max_length=32, blank=True, default="", db_index=True) + whash = models.CharField(max_length=32, blank=True, default="", db_index=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) @@ -100,6 +105,7 @@ class TempUpload(models.Model): custom_tags = models.JSONField(default=list, blank=True) custom_notes = models.TextField(blank=True, default="") iqdb_data = models.JSONField(null=True, blank=True) + visual_matches = models.JSONField(null=True, blank=True) library_item = models.ForeignKey( MediaItem, null=True, diff --git a/backend/apps/library/serializers.py b/backend/apps/library/serializers.py index 71784e1..df6b193 100644 --- a/backend/apps/library/serializers.py +++ b/backend/apps/library/serializers.py @@ -117,6 +117,7 @@ class TempUploadSerializer(serializers.ModelSerializer): "custom_tags", "custom_notes", "iqdb_data", + "visual_matches", "library_j_id", "file_url", "preview_url", diff --git a/backend/apps/library/services.py b/backend/apps/library/services.py index 1622244..32250cf 100644 --- a/backend/apps/library/services.py +++ b/backend/apps/library/services.py @@ -1,5 +1,6 @@ import hashlib import json +import logging import mimetypes import os import re @@ -7,12 +8,19 @@ import shutil import subprocess from pathlib import Path +import imagehash from django.conf import settings from django.http import FileResponse, Http404, HttpResponse from django.utils.text import get_valid_filename +from PIL import Image from .models import MediaItem, MediaLocation +logger = logging.getLogger(__name__) + +HASH_FIELDS = ("ahash", "dhash", "phash", "whash") +IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".apng", ".webp"} + ALLOWED_EXTENSIONS = { ".jpg", ".jpeg", @@ -82,6 +90,41 @@ def rename_location_to_j_id(item, location): return location +def compute_visual_hashes(path): + """Perceptual hashes for an image file (empty dict for other files).""" + path = Path(path) + if path.suffix.lower() not in IMAGE_EXTENSIONS: + return {} + try: + with Image.open(path) as image: + converted = image.convert("RGB") + return { + "ahash": str(imagehash.average_hash(converted, hash_size=8)), + "dhash": str(imagehash.dhash(converted, hash_size=8)), + "phash": str(imagehash.phash(converted, hash_size=8)), + "whash": str(imagehash.whash(converted, hash_size=8)), + } + except Exception: # noqa: BLE001 - hashing must never break indexing + logger.exception("Could not compute visual hashes for %s", path) + return {} + + +def ensure_visual_hashes(item): + """Fill in missing perceptual hashes for a media item.""" + if all(getattr(item, field) for field in HASH_FIELDS): + return item + location = item.locations.first() + if location is None: + return item + hashes = compute_visual_hashes(location.path) + if not hashes: + return item + for field, value in hashes.items(): + setattr(item, field, value) + item.save(update_fields=[*hashes.keys(), "updated_at"]) + return item + + def parse_tags(raw): """Normalize a comma-separated string or JSON list into a list of tags.""" if raw is None: diff --git a/backend/apps/library/tools.py b/backend/apps/library/tools.py new file mode 100644 index 0000000..0d3b4b7 --- /dev/null +++ b/backend/apps/library/tools.py @@ -0,0 +1,394 @@ +"""Library tooling: duplicates, deletion and storage overview.""" + +import logging +import shutil +from pathlib import Path + +from django.conf import settings +from django.core.cache import cache +from django.db.models import Count, Q +from rest_framework import status +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response +from rest_framework.views import APIView + +from .models import MediaItem, MediaLocation +from .permissions import CanUpload + +logger = logging.getLogger(__name__) + +HASH_FIELDS = ("ahash", "dhash", "phash", "whash") +STORAGE_CACHE_KEY = "j621.library.storage" +STORAGE_CACHE_TTL = 60 +GROUPS_PER_PAGE = 20 + + +def parse_threshold(value, default=0.8): + try: + parsed = float(value) + except (TypeError, ValueError): + return default + if parsed > 1: + parsed = parsed / 100.0 + return min(max(parsed, 0.0), 1.0) + + +def parse_algorithms(value): + if not isinstance(value, list): + return list(HASH_FIELDS) + selected = [field for field in value if field in HASH_FIELDS] + return selected or list(HASH_FIELDS) + + +def hashes_similarity(first_hashes, second_hashes, algorithms, threshold): + """Best similarity between two hash mappings, or None below threshold.""" + best = None + for field in algorithms: + first = first_hashes.get(field) or "" + second = second_hashes.get(field) or "" + if not first or not second: + continue + try: + distance = bin(int(first, 16) ^ int(second, 16)).count("1") + except ValueError: + continue + value = 1.0 - distance / 64.0 + if best is None or value > best: + best = value + if best is None or best < threshold: + return None + return best + + +def similarity_between(item_a, item_b, algorithms, threshold): + """Best similarity across the selected algorithms, or None below threshold.""" + return hashes_similarity( + {field: getattr(item_a, field, "") for field in algorithms}, + {field: getattr(item_b, field, "") for field in algorithms}, + algorithms, + threshold, + ) + + +def hashed_items(algorithms): + query = Q() + for field in algorithms: + query |= ~Q(**{field: ""}) + return list(MediaItem.objects.filter(query).prefetch_related("locations")) + + +def display_rating(item): + if item.rating: + return item.rating + data = item.e621_data or {} + rating = data.get("rating") if isinstance(data, dict) else None + return rating if rating in {"s", "q", "e"} else "" + + +def item_brief(item): + locations = list(item.locations.all()) + location = locations[0] if locations else None + return { + "j_id": f"J-{item.id}", + "md5": item.md5, + "filename": Path(location.rel_path).name if location else item.md5, + "size": item.size, + "rating": display_rating(item), + "location_count": len(locations), + "uploaded_by": item.uploaded_by.username if item.uploaded_by else None, + "e621_post_id": item.e621_post_id, + } + + +def resolve_item(data): + j_id = str(data.get("j_id") or "").strip() + md5 = str(data.get("md5") or "").strip().lower() + if j_id: + numeric = j_id[2:] if j_id.upper().startswith("J-") else j_id + if numeric.isdigit(): + item = MediaItem.objects.filter(pk=int(numeric)).first() + if item is not None: + return item + if md5: + return MediaItem.objects.filter(md5=md5).first() + return None + + +class ExactDuplicatesView(APIView): + """Items whose content exists at more than one path.""" + + permission_classes = [IsAuthenticated] + + def get(self, request): + items = ( + MediaItem.objects.annotate(location_count=Count("locations")) + .filter(location_count__gt=1) + .prefetch_related("locations") + .order_by("-location_count", "id") + ) + groups = [] + for item in items: + brief = item_brief(item) + brief["locations"] = [ + {"id": location.id, "rel_path": location.rel_path} + for location in item.locations.all() + ] + groups.append(brief) + return Response({"count": len(groups), "groups": groups}) + + +class VisualMatchesView(APIView): + """Items visually similar to one library item.""" + + permission_classes = [IsAuthenticated] + + def post(self, request): + threshold = parse_threshold(request.data.get("threshold")) + algorithms = parse_algorithms(request.data.get("algorithms")) + target = resolve_item(request.data) + if target is None: + return Response( + {"detail": "A j_id or md5 is required."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + matches = [] + for item in hashed_items(algorithms): + if item.pk == target.pk: + continue + similarity = similarity_between(target, item, algorithms, threshold) + if similarity is None: + continue + brief = item_brief(item) + brief["similarity"] = round(similarity * 100, 1) + matches.append(brief) + matches.sort(key=lambda entry: entry["similarity"], reverse=True) + + return Response( + { + "target": item_brief(target), + "threshold": round(threshold * 100, 1), + "algorithms": algorithms, + "count": len(matches), + "matches": matches[:200], + } + ) + + +class VisualGroupsView(APIView): + """Groups of visually similar items (union-find over perceptual hashes). + + Pairwise comparison is O(n^2) with fast bit operations, which is fine for + a personal library. Revisit with a bucketed index if libraries grow huge. + """ + + permission_classes = [IsAuthenticated] + + def post(self, request): + threshold = parse_threshold(request.data.get("threshold")) + algorithms = parse_algorithms(request.data.get("algorithms")) + page = max(1, int(request.data.get("page") or 1)) + + items = hashed_items(algorithms) + parent = list(range(len(items))) + + def find(index): + while parent[index] != index: + parent[index] = parent[parent[index]] + index = parent[index] + return index + + def union(first, second): + root_a = find(first) + root_b = find(second) + if root_a != root_b: + parent[root_b] = root_a + + for first in range(len(items)): + for second in range(first + 1, len(items)): + if find(first) == find(second): + continue + if ( + similarity_between( + items[first], items[second], algorithms, threshold + ) + is not None + ): + union(first, second) + + grouped = {} + for index, item in enumerate(items): + grouped.setdefault(find(index), []).append(item) + + groups = [members for members in grouped.values() if len(members) >= 2] + groups.sort(key=len, reverse=True) + + total = len(groups) + start = (page - 1) * GROUPS_PER_PAGE + page_groups = groups[start : start + GROUPS_PER_PAGE] + + return Response( + { + "count": total, + "page": page, + "per_page": GROUPS_PER_PAGE, + "has_next": start + GROUPS_PER_PAGE < total, + "threshold": round(threshold * 100, 1), + "algorithms": algorithms, + "groups": [ + {"size": len(members), "members": [item_brief(item) for item in members]} + for members in page_groups + ], + } + ) + + +def can_delete(user, item): + if user.is_superuser or user.role == user.ROLE_STAFF: + return True + return item.uploaded_by_id == user.id + + +def remove_watched_file(path): + """Delete a file only when it lives inside the watched folder.""" + watched = Path(settings.WATCHED_FOLDER).resolve() + try: + resolved = Path(path).resolve() + resolved.relative_to(watched) + except (ValueError, OSError): + return False + if resolved.is_file(): + resolved.unlink() + return True + return False + + +class DeleteFilesView(APIView): + """Delete items (with every copy) or individual duplicate locations.""" + + permission_classes = [CanUpload] + + def post(self, request): + j_ids = request.data.get("j_ids") or [] + location_ids = request.data.get("location_ids") or [] + if not isinstance(j_ids, list) or not isinstance(location_ids, list): + return Response( + {"detail": "j_ids and location_ids must be lists."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + deleted = [] + errors = [] + + numeric_ids = [] + for value in j_ids: + text = str(value).strip() + numeric = text[2:] if text.upper().startswith("J-") else text + if numeric.isdigit(): + numeric_ids.append(int(numeric)) + + for item in MediaItem.objects.filter(pk__in=numeric_ids).prefetch_related( + "locations" + ): + if not can_delete(request.user, item): + errors.append({"j_id": f"J-{item.id}", "error": "permission denied"}) + continue + for location in item.locations.all(): + remove_watched_file(location.path) + deleted.append(f"J-{item.id}") + item.delete() + + cache.delete(STORAGE_CACHE_KEY) + + for location in MediaLocation.objects.filter( + id__in=location_ids + ).select_related("item"): + if not can_delete(request.user, location.item): + errors.append( + {"location": location.id, "error": "permission denied"} + ) + continue + location_id = location.id + item = location.item + remove_watched_file(location.path) + location.delete() + if item.locations.exists(): + deleted.append(f"location {location_id}") + else: + deleted.append(f"J-{item.id}") + item.delete() + + return Response({"deleted": deleted, "errors": errors}) + + +class ClearTempView(APIView): + """Remove staged files from the temp upload folder.""" + + permission_classes = [CanUpload] + + def post(self, request): + temp_dir = Path(settings.MEDIA_ROOT) / "uploads" / "temp" + removed = 0 + if temp_dir.exists(): + for entry in temp_dir.iterdir(): + if entry.is_file(): + entry.unlink() + removed += 1 + cache.delete(STORAGE_CACHE_KEY) + return Response({"removed": removed}) + + +class StorageView(APIView): + """Disk usage for the watched folder, media root and temp uploads.""" + + permission_classes = [IsAuthenticated] + + def get(self, request): + cached = cache.get(STORAGE_CACHE_KEY) + if cached is not None: + return Response(cached) + + watched = Path(settings.WATCHED_FOLDER) + media_root = Path(settings.MEDIA_ROOT) + temp_dir = media_root / "uploads" / "temp" + + usage = shutil.disk_usage( + str(watched) if watched.exists() else str(Path(settings.BASE_DIR)) + ) + library_size = ( + sum(file.stat().st_size for file in watched.rglob("*") if file.is_file()) + if watched.exists() + else 0 + ) + media_size = ( + sum(file.stat().st_size for file in media_root.rglob("*") if file.is_file()) + if media_root.exists() + else 0 + ) + temp_files = ( + [file for file in temp_dir.rglob("*") if file.is_file()] + if temp_dir.exists() + else [] + ) + + payload = { + "watched_folder": { + "path": str(watched), + "total": usage.total, + "used": usage.used, + "free": usage.free, + "library_size": library_size, + "percent_used": ( + round(usage.used / usage.total * 100, 1) if usage.total else 0 + ), + }, + "media": {"path": str(media_root), "size": media_size}, + "temp": { + "path": str(temp_dir), + "size": sum(file.stat().st_size for file in temp_files), + "files": len(temp_files), + }, + "library_items": MediaItem.objects.count(), + } + cache.set(STORAGE_CACHE_KEY, payload, STORAGE_CACHE_TTL) + return Response(payload) diff --git a/backend/apps/library/uploads.py b/backend/apps/library/uploads.py index df20318..9828735 100644 --- a/backend/apps/library/uploads.py +++ b/backend/apps/library/uploads.py @@ -28,10 +28,40 @@ from . import services from .models import MediaItem, TempUpload from .permissions import CanUpload from .serializers import TempUploadSerializer +from .tools import HASH_FIELDS, hashes_similarity logger = logging.getLogger(__name__) +def find_library_matches(path, limit=10): + """Library items visually similar to a staged file.""" + hashes = services.compute_visual_hashes(path) + if not hashes: + return [] + algorithms = list(HASH_FIELDS) + threshold = settings.VISUAL_MATCH_THRESHOLD + matches = [] + for item in MediaItem.objects.prefetch_related("locations"): + similarity = hashes_similarity( + hashes, + {field: getattr(item, field, "") for field in algorithms}, + algorithms, + threshold, + ) + if similarity is None: + continue + location = item.locations.first() + matches.append( + { + "j_id": f"J-{item.id}", + "filename": Path(location.rel_path).name if location else item.md5, + "similarity": round(similarity * 100, 1), + } + ) + matches.sort(key=lambda entry: entry["similarity"], reverse=True) + return matches[:limit] + + def complete_temp_upload(temp, download_url=None): """Index the upload into the library. @@ -57,6 +87,7 @@ def complete_temp_upload(temp, download_url=None): raise item, _, location, _ = services.index_file(destination, folder) services.rename_location_to_j_id(item, location) + services.ensure_visual_hashes(item) if temp.file: temp.file.delete(save=False) else: @@ -69,6 +100,7 @@ def complete_temp_upload(temp, download_url=None): shutil.copyfileobj(source, target) item, _, location, _ = services.index_file(destination, folder) services.rename_location_to_j_id(item, location) + services.ensure_visual_hashes(item) temp.file.delete(save=False) temp.library_item = item @@ -143,6 +175,11 @@ class TempUploadViewSet( temp.resolution = TempUpload.RESOLUTION_DUPLICATE temp.library_item = existing temp.file.delete(save=False) + else: + matches = find_library_matches(temp.file.path) + if matches: + temp.visual_matches = matches + temp.status = TempUpload.STATUS_VISUAL_MATCH temp.save() return Response( self.get_serializer(temp).data, status=status.HTTP_201_CREATED diff --git a/backend/apps/library/urls.py b/backend/apps/library/urls.py index 76b7268..bb076eb 100644 --- a/backend/apps/library/urls.py +++ b/backend/apps/library/urls.py @@ -1,6 +1,14 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter +from .tools import ( + ClearTempView, + DeleteFilesView, + ExactDuplicatesView, + StorageView, + VisualGroupsView, + VisualMatchesView, +) from .uploads import TempUploadViewSet from .views import ClientDownloadView, DownloadTaskViewSet, MediaItemViewSet @@ -12,4 +20,22 @@ router.register("online/downloads", DownloadTaskViewSet, basename="download") urlpatterns = [ path("", include(router.urls)), path("online/file/", ClientDownloadView.as_view(), name="client_download"), + path( + "duplicates/md5/", + ExactDuplicatesView.as_view(), + name="duplicates_md5", + ), + path( + "duplicates/visual/", + VisualMatchesView.as_view(), + name="duplicates_visual", + ), + path( + "duplicates/visual_groups/", + VisualGroupsView.as_view(), + name="duplicates_groups", + ), + path("delete/", DeleteFilesView.as_view(), name="delete_files"), + path("temp/clear/", ClearTempView.as_view(), name="clear_temp"), + path("storage/", StorageView.as_view(), name="storage_info"), ] diff --git a/backend/apps/library/views.py b/backend/apps/library/views.py index c60fde5..b867e45 100644 --- a/backend/apps/library/views.py +++ b/backend/apps/library/views.py @@ -16,7 +16,7 @@ from rest_framework.response import Response from rest_framework.views import APIView from . import services -from .downloads import start_download_task +from .downloads import reap_stale_downloads, start_download_task from .models import DownloadTask, MediaItem from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly from .serializers import DownloadTaskSerializer, MediaItemSerializer @@ -41,6 +41,15 @@ class MediaItemViewSet( ) if not self.request.user.is_authenticated: queryset = queryset.filter(hidden_from_guests=False) + j_ids = self.request.query_params.get("j_ids", "").strip() + if j_ids: + numeric_ids = [] + for value in j_ids.split(","): + text = value.strip() + number = text[2:] if text.upper().startswith("J-") else text + if number.isdigit(): + numeric_ids.append(int(number)) + queryset = queryset.filter(pk__in=numeric_ids) search = self.request.query_params.get("search", "").strip() if search: queryset = queryset.filter(locations__rel_path__icontains=search) @@ -203,6 +212,7 @@ class DownloadTaskViewSet( http_method_names = ["get", "post", "head", "options"] def get_queryset(self): + reap_stale_downloads() queryset = DownloadTask.objects.select_related("library_item") user = self.request.user if not (user.is_staff or user.is_superuser): diff --git a/backend/config/settings.py b/backend/config/settings.py index c84b06a..261c0b6 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -166,6 +166,9 @@ GUEST_BLACKLIST_FALLBACK = [ ] GUEST_BLACKLIST_TTL = int(os.getenv("GUEST_BLACKLIST_TTL", "3600")) +# Similarity threshold for flagging staged uploads that match library items. +VISUAL_MATCH_THRESHOLD = float(os.getenv("VISUAL_MATCH_THRESHOLD", "0.9")) + # Redis cache (run via docker compose at the repo root), shared by web # workers and management commands (e.g. the mirrored guest blacklist). CACHES = { diff --git a/backend/config/urls.py b/backend/config/urls.py index b4a0b8a..b16204f 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -6,6 +6,7 @@ from django.urls import include, path urlpatterns = [ path("admin/", admin.site.urls), path("api/auth/", include("apps.accounts.urls")), + path("api/", include("apps.accounts.api_urls")), path("api/", include("apps.core.urls")), path("api/", include("apps.library.urls")), ] diff --git a/backend/requirements.txt b/backend/requirements.txt index dc6ddaf..9bc1e3b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -4,5 +4,6 @@ django-filter>=25.1 Pillow>=11.0 python-dotenv>=1.0 requests>=2.32 +imagehash>=4.3 PyMySQL>=1.1 redis>=5.0 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 0546564..acd5442 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -13,10 +13,13 @@ import AccountPage from "@/features/account/AccountPage"; import LoginPage from "@/features/auth/LoginPage"; import RegisterPage from "@/features/auth/RegisterPage"; import DetailPage from "@/features/detail/DetailPage"; +import DeletePage from "@/features/library/DeletePage"; +import DuplicatesPage from "@/features/library/DuplicatesPage"; import LibraryPage from "@/features/library/LibraryPage"; import Md5Redirect from "@/features/library/Md5Redirect"; import OnlinePage from "@/features/online/OnlinePage"; import UploadPage from "@/features/upload/UploadPage"; +import UsersPage from "@/features/users/UsersPage"; import { useAuth } from "@/store/auth"; function RequireAuth({ children }: { children: ReactNode }) { @@ -53,6 +56,30 @@ export default function App() { } /> } /> } /> + + + + } + /> + + + + } + /> + + + + } + /> - - {user.username.slice(0, 1).toUpperCase()} - + {user.avatar_url ? ( + {user.username} + ) : ( + + {user.username.slice(0, 1).toUpperCase()} + + )} {user.username} diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index f4b8683..cf6d849 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -1,4 +1,5 @@ import { + Copy, FolderOpen, Globe, LogIn, @@ -6,7 +7,9 @@ import { Menu, Search, Settings, + Trash2, Upload, + Users, type LucideIcon, } from "lucide-react"; import { useMemo, useState } from "react"; @@ -59,6 +62,18 @@ function CommandPaletteDialog({ onClose }: { onClose: () => void }) { icon: Upload, run: () => navigate("/upload"), }, + { + id: "duplicates", + label: "Find duplicates", + icon: Copy, + run: () => navigate("/duplicates"), + }, + { + id: "delete", + label: "Delete files & storage", + icon: Trash2, + run: () => navigate("/delete"), + }, ] : []), { @@ -94,6 +109,14 @@ function CommandPaletteDialog({ onClose }: { onClose: () => void }) { icon: Settings, run: () => navigate("/account"), }); + if (user.is_staff || user.is_superuser || user.role === "staff") { + list.push({ + id: "users", + label: "Manage users", + icon: Users, + run: () => navigate("/users"), + }); + } list.push({ id: "logout", label: `Log out (${user.username})`, diff --git a/frontend/src/features/library/DeletePage.tsx b/frontend/src/features/library/DeletePage.tsx new file mode 100644 index 0000000..6725447 --- /dev/null +++ b/frontend/src/features/library/DeletePage.tsx @@ -0,0 +1,271 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; +import { Link } from "react-router-dom"; + +import { Button, EmptyState, Spinner, inputClass } from "@/components/ui"; +import { api, errorMessage } from "@/lib/api"; +import { cn } from "@/lib/cn"; +import { formatBytes } from "@/lib/format"; +import type { MediaItem, Paginated, StorageInfo } from "@/lib/types"; + +function parseJIds(text: string): string[] { + return text + .split(/[\s,]+/) + .map((value) => value.trim().toUpperCase()) + .filter((value) => /^J-\d+$/.test(value)); +} + +export default function DeletePage() { + const queryClient = useQueryClient(); + const [jIdsInput, setJIdsInput] = useState(""); + const [loadedIds, setLoadedIds] = useState([]); + const [selected, setSelected] = useState([]); + const [status, setStatus] = useState(null); + + const storageQuery = useQuery({ + queryKey: ["storage"], + queryFn: () => api("/api/storage/"), + }); + + const filesQuery = useQuery({ + queryKey: ["delete-files", loadedIds], + queryFn: () => + api>(`/api/files/?j_ids=${loadedIds.join(",")}`), + enabled: loadedIds.length > 0, + }); + + const deleteMutation = useMutation({ + mutationFn: (jIds: string[]) => + api<{ deleted: string[]; errors: { j_id?: string; error: string }[] }>( + "/api/delete/", + { method: "POST", json: { j_ids: jIds } }, + ), + onSuccess: (result) => { + setSelected([]); + setLoadedIds([]); + setStatus( + `Deleted ${result.deleted.length} item(s)${ + result.errors.length > 0 ? ` · ${result.errors.length} failed` : "" + }.`, + ); + void queryClient.invalidateQueries({ queryKey: ["storage"] }); + void queryClient.invalidateQueries({ queryKey: ["files"] }); + void queryClient.invalidateQueries({ queryKey: ["duplicates-md5"] }); + }, + onError: (error) => setStatus(errorMessage(error)), + }); + + const tempMutation = useMutation({ + mutationFn: () => + api<{ removed: number }>("/api/temp/clear/", { method: "POST" }), + onSuccess: (result) => { + setStatus(`Removed ${result.removed} temp file(s).`); + void queryClient.invalidateQueries({ queryKey: ["storage"] }); + }, + onError: (error) => setStatus(errorMessage(error)), + }); + + function toggleSelected(jId: string) { + setSelected((current) => + current.includes(jId) + ? current.filter((value) => value !== jId) + : [...current, jId], + ); + } + + const storage = storageQuery.data; + + return ( +
+
+

Delete & storage

+

+ Inspect disk usage and remove files from the library. +

+
+ + {status ? ( +

+ {status} +

+ ) : null} + +
+ {storageQuery.isPending ? ( + + ) : storage ? ( + <> +
+

+ Library size +

+

+ {formatBytes(storage.watched_folder.library_size)} +

+

+ {storage.watched_folder.path} +

+

+ {storage.library_items} items +

+
+
+

+ Disk +

+

+ {formatBytes(storage.watched_folder.used)} /{" "} + {formatBytes(storage.watched_folder.total)} +

+
+
= 95 + ? "bg-ctp-red" + : storage.watched_folder.percent_used >= 80 + ? "bg-ctp-peach" + : "bg-ctp-blue", + )} + style={{ width: `${storage.watched_folder.percent_used}%` }} + /> +
+

+ {storage.watched_folder.percent_used}% used ·{" "} + {formatBytes(storage.watched_folder.free)} free +

+
+
+

+ Media folder +

+

+ {formatBytes(storage.media.size)} +

+

+ {storage.media.path} +

+
+
+

+ Temp uploads +

+

+ {formatBytes(storage.temp.size)} · {storage.temp.files} file(s) +

+ +
+ + ) : ( + + )} +
+ +
+

+ Delete files by J-ID +

+