Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af378e7d71 | ||
|
|
59f397a96c | ||
|
|
9ababb8b48 | ||
|
|
37085c5dac | ||
|
|
ecac4cb8b4 |
+20
-7
@@ -162,8 +162,7 @@ jobs:
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
EXISTING="$(curl -sf -H "$AUTH" "$API/releases/$RELEASE_ID/assets" \
|
EXISTING="$(curl -sf -H "$AUTH" "$API/releases/$RELEASE_ID/assets" \
|
||||||
| python3 -c 'import json,sys; print("\n".join(a["name"] for a in json.load(sys.stdin)))' \
|
|| echo '[]')"
|
||||||
|| true)"
|
|
||||||
for FILE in desktop/release/*"$VERSION"*.deb \
|
for FILE in desktop/release/*"$VERSION"*.deb \
|
||||||
desktop/release/*"$VERSION"*.pkg.tar.zst \
|
desktop/release/*"$VERSION"*.pkg.tar.zst \
|
||||||
desktop/release/latest-linux.yml \
|
desktop/release/latest-linux.yml \
|
||||||
@@ -172,11 +171,25 @@ jobs:
|
|||||||
desktop/release/*"$VERSION"*.exe.blockmap; do
|
desktop/release/*"$VERSION"*.exe.blockmap; do
|
||||||
[ -e "$FILE" ] || continue
|
[ -e "$FILE" ] || continue
|
||||||
NAME="$(basename "$FILE")"
|
NAME="$(basename "$FILE")"
|
||||||
case "$EXISTING" in
|
# Replace the asset when it is already there: latest*.yml must
|
||||||
*"$NAME"*) echo " already attached: $NAME"; continue ;;
|
# reference the installers built by *this* run (NSIS builds are
|
||||||
esac
|
# not bit-reproducible), so old copies are deleted first.
|
||||||
echo " attaching $NAME"
|
ASSET_ID="$(printf '%s' "$EXISTING" | python3 -c '
|
||||||
|
import json, sys
|
||||||
|
name = sys.argv[1]
|
||||||
|
print(next((str(a["id"]) for a in json.load(sys.stdin) if a["name"] == name), ""))
|
||||||
|
' "$NAME")"
|
||||||
|
if [ -n "$ASSET_ID" ]; then
|
||||||
|
echo " replacing $NAME"
|
||||||
|
curl -sf -X DELETE -H "$AUTH" \
|
||||||
|
"$API/releases/$RELEASE_ID/assets/$ASSET_ID" >/dev/null
|
||||||
|
else
|
||||||
|
echo " attaching $NAME"
|
||||||
|
fi
|
||||||
|
# Names like "J621 Setup 0.1.1.exe" contain spaces: encode them
|
||||||
|
# or curl refuses the URL (exit 3).
|
||||||
|
ENCODED="$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$NAME")"
|
||||||
curl -sf -X POST -H "$AUTH" -H "Content-Type: application/octet-stream" \
|
curl -sf -X POST -H "$AUTH" -H "Content-Type: application/octet-stream" \
|
||||||
--data-binary @"$FILE" "$API/releases/$RELEASE_ID/assets?name=$NAME" >/dev/null
|
--data-binary @"$FILE" "$API/releases/$RELEASE_ID/assets?name=$ENCODED" >/dev/null
|
||||||
done
|
done
|
||||||
echo "Release: ${{ github.server_url }}/${{ github.repository }}/releases/tag/$TAG"
|
echo "Release: ${{ github.server_url }}/${{ github.repository }}/releases/tag/$TAG"
|
||||||
|
|||||||
+14
-3
@@ -20,6 +20,13 @@ they land.
|
|||||||
- [x] Tag cloud in the sidebar (click to search, hidden from guests for
|
- [x] Tag cloud in the sidebar (click to search, hidden from guests for
|
||||||
blacklisted items)
|
blacklisted items)
|
||||||
- [x] Status filter (matched / not_found / deleted / custom / unknown)
|
- [x] Status filter (matched / not_found / deleted / custom / unknown)
|
||||||
|
- [x] **Cacheable media serving**
|
||||||
|
- [x] Stable signed URLs (7 d TTL, 24 h rotation) plus a `v=<md5>` cache
|
||||||
|
buster, so replacing a file under the same J-ID invalidates it
|
||||||
|
- [x] ETag/Last-Modified and a private `Cache-Control` (immutable for
|
||||||
|
versioned media), conditional 304s
|
||||||
|
- [x] Real 480 px image thumbnails (cached by MD5) instead of serving
|
||||||
|
full-size originals through the thumbnail endpoint
|
||||||
|
|
||||||
- [x] **Random image** endpoint and page: `GET /api/random/` (aliases
|
- [x] **Random image** endpoint and page: `GET /api/random/` (aliases
|
||||||
`/random`, `/random/`) with `rating=s,q,e` filters; fastfetch mode
|
`/random`, `/random/`) with `rating=s,q,e` filters; fastfetch mode
|
||||||
@@ -104,12 +111,16 @@ Files now stage first and are resolved before entering the library.
|
|||||||
shell indicator; rate-limited runs retry with backoff
|
shell indicator; rate-limited runs retry with backoff
|
||||||
- [x] Perceptual-hash comparison against the library, loaded once per batch
|
- [x] Perceptual-hash comparison against the library, loaded once per batch
|
||||||
- [x] IQDB candidates stored with one batched enrichment request
|
- [x] IQDB candidates stored with one batched enrichment request
|
||||||
|
- [x] Indexed uploads are announced through a bounded per-run feed and the
|
||||||
|
staged row is deleted; nothing persists on the board to dismiss
|
||||||
- [x] **Upload UI**
|
- [x] **Upload UI**
|
||||||
- [x] Three-column board: Pending & Unmatched / Visual Similarity Detected /
|
- [x] Four-column board: Pending & Unmatched / Visual Similarity Detected /
|
||||||
Auto-uploaded & Indexed, private per user (staff included)
|
Auto-uploaded & Indexed (session feed) / Failed, private per user
|
||||||
|
(staff included)
|
||||||
- [x] Metadata modal (link to e621 post, IQDB candidates, custom metadata)
|
- [x] Metadata modal (link to e621 post, IQDB candidates, custom metadata)
|
||||||
- [x] Per-file progress plus background pipeline status
|
- [x] Per-file progress plus background pipeline status
|
||||||
- [x] Bulk actions: bulk rate, discard all (pending/visual), dismiss all
|
- [x] Bulk actions: bulk rate and discard all (chunked past the API's
|
||||||
|
1000-id cap)
|
||||||
|
|
||||||
## 4. Staff tools
|
## 4. Staff tools
|
||||||
|
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from apps.library.models import (
|
|||||||
TempUpload,
|
TempUpload,
|
||||||
)
|
)
|
||||||
from apps.library.services import MEDIA_FILE_SALT
|
from apps.library.services import MEDIA_FILE_SALT
|
||||||
|
from apps.library.signing_urls import sign_payload
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
|
|
||||||
@@ -122,21 +123,15 @@ class SecurityTestCase(TestCase):
|
|||||||
return item
|
return item
|
||||||
|
|
||||||
def old_signature(self, item, action="raw", age=3 * 86400):
|
def old_signature(self, item, action="raw", age=3 * 86400):
|
||||||
"""A valid signature minted `age` seconds ago."""
|
"""A signed media URL whose expiry is `age` seconds in the past."""
|
||||||
real_time = signing.time
|
return signing.Signer(salt=MEDIA_FILE_SALT).sign_object(
|
||||||
|
{
|
||||||
class Backdated:
|
"item": item.id,
|
||||||
def time(self):
|
"user": self.users["sec-uploader"].id,
|
||||||
return real_time.time() - age
|
"action": action,
|
||||||
|
"exp": int(time.time()) - age,
|
||||||
try:
|
}
|
||||||
signing.time = Backdated()
|
)
|
||||||
return signing.dumps(
|
|
||||||
{"item": item.id, "user": self.users["sec-uploader"].id, "action": action},
|
|
||||||
salt=MEDIA_FILE_SALT,
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
signing.time = real_time
|
|
||||||
|
|
||||||
|
|
||||||
class GuestVisibilityTests(SecurityTestCase):
|
class GuestVisibilityTests(SecurityTestCase):
|
||||||
@@ -183,9 +178,9 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
def test_authenticated_users_and_signed_urls_see_protected_items(self):
|
def test_authenticated_users_and_signed_urls_see_protected_items(self):
|
||||||
uploader = self.client_for("sec-uploader")
|
uploader = self.client_for("sec-uploader")
|
||||||
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
|
self.assertEqual(uploader.get(f"/api/files/J-{self.hidden.id}/").status_code, 200)
|
||||||
signed = signing.dumps(
|
signed = sign_payload(
|
||||||
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
||||||
salt=MEDIA_FILE_SALT,
|
MEDIA_FILE_SALT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
|
self.guest.get(f"/api/files/J-{self.hidden.id}/raw/?sig={signed}").status_code,
|
||||||
@@ -193,9 +188,9 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_signature_integrity(self):
|
def test_signature_integrity(self):
|
||||||
signed = signing.dumps(
|
signed = sign_payload(
|
||||||
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
{"item": self.hidden.id, "user": self.users["sec-uploader"].id, "action": "raw"},
|
||||||
salt=MEDIA_FILE_SALT,
|
MEDIA_FILE_SALT,
|
||||||
)
|
)
|
||||||
raw = f"/api/files/J-{self.hidden.id}/raw/"
|
raw = f"/api/files/J-{self.hidden.id}/raw/"
|
||||||
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
|
thumbnail = f"/api/files/J-{self.hidden.id}/thumbnail/"
|
||||||
@@ -203,10 +198,29 @@ class GuestVisibilityTests(SecurityTestCase):
|
|||||||
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
|
self.assertEqual(self.guest.get(f"{raw}?sig={signed[:-4]}AAAA").status_code, 404)
|
||||||
# Valid signature, wrong action.
|
# Valid signature, wrong action.
|
||||||
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
|
self.assertEqual(self.guest.get(f"{thumbnail}?sig={signed}").status_code, 404)
|
||||||
# Expired signature (minted three days ago).
|
# Expired signature (expiry three days ago).
|
||||||
expired = self.old_signature(self.hidden)
|
expired = self.old_signature(self.hidden)
|
||||||
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
|
self.assertEqual(self.guest.get(f"{raw}?sig={expired}").status_code, 404)
|
||||||
|
|
||||||
|
def test_signed_media_urls_are_stable_and_versioned(self):
|
||||||
|
"""The same item must keep the same URL across responses.
|
||||||
|
|
||||||
|
A per-second signature made browsers re-download every image on every
|
||||||
|
poll; the MD5 version parameter busts caches only when the file itself
|
||||||
|
changes (the optimize flow rewrites files under the same J-ID).
|
||||||
|
"""
|
||||||
|
item = self.visible
|
||||||
|
first = services.signed_media_url(item, self.users["sec-uploader"])
|
||||||
|
time.sleep(1.1)
|
||||||
|
second = services.signed_media_url(item, self.users["sec-uploader"])
|
||||||
|
self.assertEqual(first, second)
|
||||||
|
self.assertIn(f"v={item.md5}", first)
|
||||||
|
MediaItem.objects.filter(pk=item.pk).update(md5="b" * 32)
|
||||||
|
item.refresh_from_db()
|
||||||
|
self.assertNotEqual(
|
||||||
|
services.signed_media_url(item, self.users["sec-uploader"]), first
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RoleBoundaryTests(SecurityTestCase):
|
class RoleBoundaryTests(SecurityTestCase):
|
||||||
def test_non_uploader_is_read_only(self):
|
def test_non_uploader_is_read_only(self):
|
||||||
@@ -451,9 +465,9 @@ class ThrottleTests(SecurityTestCase):
|
|||||||
md5=hashlib.md5(b"throttle-temp").hexdigest(),
|
md5=hashlib.md5(b"throttle-temp").hexdigest(),
|
||||||
size=6,
|
size=6,
|
||||||
)
|
)
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
|
{"temp": str(temp.id), "user": self.users["sec-uploader"].id},
|
||||||
salt=services.UPLOAD_FILE_SALT,
|
services.UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
|
url = f"/api/uploads/{temp.id}/file/?sig={signature}"
|
||||||
codes = {self.guest.get(url).status_code for _ in range(150)}
|
codes = {self.guest.get(url).status_code for _ in range(150)}
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# Generated by Django 6.1.1 on 2026-09-23
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
def purge_completed_uploads(apps, schema_editor):
|
||||||
|
"""Completed staged uploads are notifications, not records.
|
||||||
|
|
||||||
|
The board used to keep every auto-uploaded/indexed row until it was
|
||||||
|
dismissed by hand, so they accumulated without bound (and bulk dismissal
|
||||||
|
is capped at 1000 ids). Completions now live in a small per-run feed, so
|
||||||
|
the old rows are removed here.
|
||||||
|
"""
|
||||||
|
TempUpload = apps.get_model("library", "TempUpload")
|
||||||
|
for temp in TempUpload.objects.filter(status="completed").iterator():
|
||||||
|
if temp.file:
|
||||||
|
temp.file.delete(save=False)
|
||||||
|
temp.delete()
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("library", "0011_upload_pipeline"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="uploadrun",
|
||||||
|
name="recent_completions",
|
||||||
|
field=models.JSONField(blank=True, default=list),
|
||||||
|
),
|
||||||
|
migrations.RunPython(purge_completed_uploads, migrations.RunPython.noop),
|
||||||
|
]
|
||||||
@@ -229,6 +229,11 @@ class UploadRun(models.Model):
|
|||||||
matched = models.IntegerField(default=0)
|
matched = models.IntegerField(default=0)
|
||||||
failed = models.IntegerField(default=0)
|
failed = models.IntegerField(default=0)
|
||||||
error = models.TextField(blank=True, default="")
|
error = models.TextField(blank=True, default="")
|
||||||
|
# Rolling feed of recent completions for the upload board. Completed
|
||||||
|
# staged uploads are deleted as soon as they are indexed; this only tells
|
||||||
|
# the live page "filename -> J-x" while it watches. Bounded, never
|
||||||
|
# dismissed, and ignored by fresh page loads.
|
||||||
|
recent_completions = models.JSONField(default=list, blank=True)
|
||||||
started_at = models.DateTimeField(null=True, blank=True)
|
started_at = models.DateTimeField(null=True, blank=True)
|
||||||
updated_at = models.DateTimeField(auto_now=True)
|
updated_at = models.DateTimeField(auto_now=True)
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from datetime import timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
@@ -20,6 +19,7 @@ from .services import (
|
|||||||
VIDEO_EXTENSIONS,
|
VIDEO_EXTENSIONS,
|
||||||
signed_media_url,
|
signed_media_url,
|
||||||
)
|
)
|
||||||
|
from .signing_urls import sign_payload
|
||||||
|
|
||||||
|
|
||||||
class MediaLocationSerializer(serializers.ModelSerializer):
|
class MediaLocationSerializer(serializers.ModelSerializer):
|
||||||
@@ -199,9 +199,9 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
user = self._request_user()
|
user = self._request_user()
|
||||||
if user is None:
|
if user is None:
|
||||||
return None
|
return None
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"temp": str(obj.id), "user": user.id},
|
{"temp": str(obj.id), "user": user.id},
|
||||||
salt=UPLOAD_FILE_SALT,
|
UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
|
url = f"/api/uploads/{obj.id}/file/?sig={signature}"
|
||||||
request = self.context.get("request")
|
request = self.context.get("request")
|
||||||
@@ -339,9 +339,9 @@ class SimilarityCheckSerializer(serializers.ModelSerializer):
|
|||||||
user = self._request_user()
|
user = self._request_user()
|
||||||
if user is None or not obj.file:
|
if user is None or not obj.file:
|
||||||
return None
|
return None
|
||||||
signature = signing.dumps(
|
signature = sign_payload(
|
||||||
{"check": str(obj.id), "user": user.id},
|
{"check": str(obj.id), "user": user.id},
|
||||||
salt=UPLOAD_FILE_SALT,
|
UPLOAD_FILE_SALT,
|
||||||
)
|
)
|
||||||
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
|
url = f"/api/similarity/{obj.id}/file/?sig={signature}"
|
||||||
request = self.context.get("request")
|
request = self.context.get("request")
|
||||||
|
|||||||
@@ -6,16 +6,20 @@ import os
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
import imagehash
|
import imagehash
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.http import FileResponse, Http404, HttpResponse
|
from django.http import FileResponse, Http404, HttpResponse
|
||||||
|
from django.utils.cache import get_conditional_response
|
||||||
|
from django.utils.http import http_date
|
||||||
from django.utils.text import get_valid_filename
|
from django.utils.text import get_valid_filename
|
||||||
from PIL import Image
|
from PIL import Image, ImageOps
|
||||||
|
|
||||||
from .models import MediaItem, MediaLocation
|
from .models import MediaItem, MediaLocation
|
||||||
|
from .signing_urls import sign_payload
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -37,6 +41,11 @@ UPLOAD_FILE_SALT = "j621.upload-file"
|
|||||||
MEDIA_FILE_SALT = "j621.media-file"
|
MEDIA_FILE_SALT = "j621.media-file"
|
||||||
CHUNK_SIZE = 1024 * 1024
|
CHUNK_SIZE = 1024 * 1024
|
||||||
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
|
||||||
|
# Versioned media URLs are immutable, so they may sit in the browser cache for
|
||||||
|
# as long as the signature is guaranteed to stay valid (7 days).
|
||||||
|
MEDIA_CACHE_SECONDS = 6 * 86400
|
||||||
|
# Staged uploads and similarity files can be deleted at any moment.
|
||||||
|
TEMP_CACHE_SECONDS = 3600
|
||||||
|
|
||||||
|
|
||||||
def compute_md5(path):
|
def compute_md5(path):
|
||||||
@@ -78,14 +87,24 @@ def signed_media_url(item, user, action="raw", request=None):
|
|||||||
|
|
||||||
With a ``request`` the URL is absolute, so the SPA also works when it is
|
With a ``request`` the URL is absolute, so the SPA also works when it is
|
||||||
served from a different origin; without one it stays relative.
|
served from a different origin; without one it stays relative.
|
||||||
|
|
||||||
|
The ``v`` parameter is the item's MD5: it busts the browser cache exactly
|
||||||
|
when the file is replaced (the optimize flow rewrites files under the same
|
||||||
|
J-ID), which is what lets the URL be cached for days instead of re-minted
|
||||||
|
on every response.
|
||||||
"""
|
"""
|
||||||
path = f"/api/files/J-{item.id}/{action}/"
|
path = f"/api/files/J-{item.id}/{action}/"
|
||||||
|
params = {}
|
||||||
if user is not None and getattr(user, "is_authenticated", False):
|
if user is not None and getattr(user, "is_authenticated", False):
|
||||||
signature = signing.dumps(
|
params = {
|
||||||
{"item": item.id, "user": user.id, "action": action},
|
"v": item.md5,
|
||||||
salt=MEDIA_FILE_SALT,
|
"sig": sign_payload(
|
||||||
)
|
{"item": item.id, "user": user.id, "action": action},
|
||||||
path = f"{path}?sig={signature}"
|
MEDIA_FILE_SALT,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if params:
|
||||||
|
path = f"{path}?{urlencode(params)}"
|
||||||
if request is None:
|
if request is None:
|
||||||
return path
|
return path
|
||||||
return request.build_absolute_uri(path)
|
return request.build_absolute_uri(path)
|
||||||
@@ -207,12 +226,36 @@ class RangeFileWrapper:
|
|||||||
self.file.close()
|
self.file.close()
|
||||||
|
|
||||||
|
|
||||||
def serve_file(request, path, download=False):
|
def _apply_cache_headers(response, cache_control, etag, mtime):
|
||||||
"""Serve a file with HTTP range support (needed for video seeking)."""
|
response["Cache-Control"] = cache_control
|
||||||
|
response["ETag"] = etag
|
||||||
|
response["Last-Modified"] = http_date(mtime)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def serve_file(request, path, download=False, *, max_age=TEMP_CACHE_SECONDS, immutable=False):
|
||||||
|
"""Serve a file with HTTP range support (needed for video seeking).
|
||||||
|
|
||||||
|
Responses carry validators (ETag/Last-Modified) and a private
|
||||||
|
``Cache-Control`` so browsers reuse media instead of re-downloading it on
|
||||||
|
every SPA poll. ``max_age``/``immutable`` are chosen by the caller: versioned
|
||||||
|
library media can be cached hard, staged files only briefly.
|
||||||
|
"""
|
||||||
path = Path(path)
|
path = Path(path)
|
||||||
if not path.is_file():
|
if not path.is_file():
|
||||||
raise Http404
|
raise Http404
|
||||||
size = path.stat().st_size
|
stat = path.stat()
|
||||||
|
size = stat.st_size
|
||||||
|
etag = f'W/"{size:x}-{stat.st_mtime_ns:x}"'
|
||||||
|
last_modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||||
|
conditional = get_conditional_response(
|
||||||
|
request, etag=etag, last_modified=last_modified
|
||||||
|
)
|
||||||
|
if conditional is not None:
|
||||||
|
return conditional
|
||||||
|
cache_control = f"private, max-age={int(max_age)}"
|
||||||
|
if immutable:
|
||||||
|
cache_control += ", immutable"
|
||||||
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
|
||||||
range_header = request.headers.get("Range", "").strip()
|
range_header = request.headers.get("Range", "").strip()
|
||||||
if range_header:
|
if range_header:
|
||||||
@@ -240,7 +283,9 @@ def serve_file(request, path, download=False):
|
|||||||
response["Content-Length"] = str(length)
|
response["Content-Length"] = str(length)
|
||||||
response["Content-Range"] = f"bytes {start}-{end}/{size}"
|
response["Content-Range"] = f"bytes {start}-{end}/{size}"
|
||||||
response["Accept-Ranges"] = "bytes"
|
response["Accept-Ranges"] = "bytes"
|
||||||
return response
|
return _apply_cache_headers(
|
||||||
|
response, cache_control, etag, stat.st_mtime
|
||||||
|
)
|
||||||
response = FileResponse(
|
response = FileResponse(
|
||||||
open(path, "rb"),
|
open(path, "rb"),
|
||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
@@ -248,7 +293,7 @@ def serve_file(request, path, download=False):
|
|||||||
filename=path.name,
|
filename=path.name,
|
||||||
)
|
)
|
||||||
response["Accept-Ranges"] = "bytes"
|
response["Accept-Ranges"] = "bytes"
|
||||||
return response
|
return _apply_cache_headers(response, cache_control, etag, stat.st_mtime)
|
||||||
|
|
||||||
|
|
||||||
class DownloadCancelled(Exception):
|
class DownloadCancelled(Exception):
|
||||||
@@ -465,3 +510,31 @@ def generate_video_thumbnail(md5, path):
|
|||||||
except (subprocess.SubprocessError, OSError):
|
except (subprocess.SubprocessError, OSError):
|
||||||
return None
|
return None
|
||||||
return target if target.exists() else None
|
return target if target.exists() else None
|
||||||
|
|
||||||
|
|
||||||
|
def generate_image_thumbnail(md5, path):
|
||||||
|
"""Downscale an image, cached under MEDIA_ROOT/thumbs like video thumbs.
|
||||||
|
|
||||||
|
The thumbnail action used to serve full-size originals for images; a
|
||||||
|
cached 480px JPEG keeps the library grid light without touching the
|
||||||
|
original file. Returns ``None`` when Pillow cannot decode the format, so
|
||||||
|
callers can fall back to the original.
|
||||||
|
"""
|
||||||
|
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
|
||||||
|
thumbs_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
target = thumbs_dir / f"{md5}.jpg"
|
||||||
|
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
|
||||||
|
return target
|
||||||
|
try:
|
||||||
|
with Image.open(path) as image:
|
||||||
|
# Animated formats: the first frame is the preview.
|
||||||
|
image.seek(0)
|
||||||
|
frame = ImageOps.exif_transpose(image) or image
|
||||||
|
frame = frame.convert("RGB")
|
||||||
|
frame.thumbnail((480, 480))
|
||||||
|
frame.save(target, "JPEG", quality=82, optimize=True)
|
||||||
|
except Exception: # noqa: BLE001 - previews must never break serving
|
||||||
|
logger.exception("Could not build an image thumbnail for %s", path)
|
||||||
|
target.unlink(missing_ok=True)
|
||||||
|
return None
|
||||||
|
return target if target.exists() else None
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""Stable, expiring signatures for media URLs.
|
||||||
|
|
||||||
|
The SPA loads media with ``<img>``/``<video>`` tags, which cannot send the
|
||||||
|
API's ``Authorization`` header, so those URLs carry a signature instead. The
|
||||||
|
signature has to be *stable*: a URL that changes on every response makes the
|
||||||
|
browser treat every refetch as a new resource and re-download the file.
|
||||||
|
|
||||||
|
URLs are signed with a plain ``Signer`` (no per-second timestamp) plus an
|
||||||
|
explicit ``exp`` claim quantized to a bucket, so every request inside a bucket
|
||||||
|
mints the exact same URL. The URL rotates once per bucket and is valid for at
|
||||||
|
least ``URL_TTL_SECONDS`` and at most ``URL_TTL_SECONDS + URL_BUCKET_SECONDS``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from django.core import signing
|
||||||
|
|
||||||
|
URL_TTL_SECONDS = 7 * 86400
|
||||||
|
URL_BUCKET_SECONDS = 24 * 3600
|
||||||
|
_BUCKETS = URL_TTL_SECONDS // URL_BUCKET_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def _expiry(now=None):
|
||||||
|
current = time.time() if now is None else now
|
||||||
|
bucket = int(current // URL_BUCKET_SECONDS)
|
||||||
|
return (bucket + _BUCKETS + 1) * URL_BUCKET_SECONDS
|
||||||
|
|
||||||
|
|
||||||
|
def sign_payload(payload, salt, now=None):
|
||||||
|
"""Sign a payload with a stable, bucket-quantized expiry."""
|
||||||
|
return signing.Signer(salt=salt).sign_object(
|
||||||
|
{**payload, "exp": _expiry(now)}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_payload(signature, salt, legacy_max_age=86400):
|
||||||
|
"""Verify a signed payload; ``None`` when missing, tampered with or expired.
|
||||||
|
|
||||||
|
Signatures minted before the stable scheme (``TimestampSigner``) are still
|
||||||
|
accepted for one release so pages open across the deploy keep working.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = signing.Signer(salt=salt).unsign_object(signature)
|
||||||
|
except signing.BadSignature:
|
||||||
|
try:
|
||||||
|
return signing.TimestampSigner(salt=salt).unsign_object(
|
||||||
|
signature, max_age=legacy_max_age
|
||||||
|
)
|
||||||
|
except signing.BadSignature:
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
expired = int(data.get("exp", 0)) < time.time()
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return None if expired else data
|
||||||
@@ -11,7 +11,6 @@ from datetime import timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from rest_framework import mixins, status, viewsets
|
from rest_framework import mixins, status, viewsets
|
||||||
from rest_framework.decorators import action
|
from rest_framework.decorators import action
|
||||||
@@ -22,6 +21,7 @@ from rest_framework.response import Response
|
|||||||
from . import services
|
from . import services
|
||||||
from .models import MediaItem, SimilarityCheck
|
from .models import MediaItem, SimilarityCheck
|
||||||
from .serializers import SimilarityCheckSerializer
|
from .serializers import SimilarityCheckSerializer
|
||||||
|
from .signing_urls import load_payload
|
||||||
from .tools import item_brief
|
from .tools import item_brief
|
||||||
from .uploads import find_library_matches
|
from .uploads import find_library_matches
|
||||||
|
|
||||||
@@ -149,14 +149,7 @@ class SimilarityCheckViewSet(
|
|||||||
check = self.get_queryset().filter(pk=pk).first()
|
check = self.get_queryset().filter(pk=pk).first()
|
||||||
else:
|
else:
|
||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
payload = None
|
payload = load_payload(signature, services.UPLOAD_FILE_SALT) if signature else None
|
||||||
if signature:
|
|
||||||
try:
|
|
||||||
payload = signing.loads(
|
|
||||||
signature, salt=services.UPLOAD_FILE_SALT, max_age=86400
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
payload = None
|
|
||||||
if payload and payload.get("check") == str(pk):
|
if payload and payload.get("check") == str(pk):
|
||||||
check = SimilarityCheck.objects.filter(pk=pk).first()
|
check = SimilarityCheck.objects.filter(pk=pk).first()
|
||||||
if check is None or not check.file:
|
if check is None or not check.file:
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
"""Signed media URLs must be stable, versioned and cacheable.
|
||||||
|
|
||||||
|
Regression: signatures embedded the current second, so every API response
|
||||||
|
re-minted every URL and browsers re-downloaded each image on every poll; the
|
||||||
|
file responses also carried no cache headers at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
|
from django.test import Client, TestCase, override_settings
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from rest_framework.authtoken.models import Token
|
||||||
|
|
||||||
|
from apps.library import services
|
||||||
|
from apps.library.models import MediaItem, MediaLocation, TempUpload
|
||||||
|
from apps.library.signing_urls import sign_payload
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
TINY_PNG = base64.b64decode(
|
||||||
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def png_bytes(width=1200, height=800, color=(20, 120, 200)):
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
Image.new("RGB", (width, height), color).save(buffer, format="PNG")
|
||||||
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class MediaCacheTests(TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
super().setUpClass()
|
||||||
|
cls._tmp = tempfile.mkdtemp(prefix="j621-cache-")
|
||||||
|
cls._media = Path(cls._tmp) / "media"
|
||||||
|
cls._watched = cls._media / "library"
|
||||||
|
cls._watched.mkdir(parents=True, exist_ok=True)
|
||||||
|
cls._settings = override_settings(
|
||||||
|
MEDIA_ROOT=str(cls._media), WATCHED_FOLDER=str(cls._watched)
|
||||||
|
)
|
||||||
|
cls._settings.enable()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls._settings.disable()
|
||||||
|
shutil.rmtree(cls._tmp, ignore_errors=True)
|
||||||
|
super().tearDownClass()
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.user = User.objects.create_user(
|
||||||
|
username="cache-uploader", password="cache-pass-123456"
|
||||||
|
)
|
||||||
|
self.user.role = "uploader"
|
||||||
|
self.user.save(update_fields=["role"])
|
||||||
|
self.token = Token.objects.create(user=self.user).key
|
||||||
|
payload = png_bytes()
|
||||||
|
path = self._watched / "cache-image.png"
|
||||||
|
path.write_bytes(payload)
|
||||||
|
self.item = MediaItem.objects.create(
|
||||||
|
md5=hashlib.md5(payload).hexdigest(), size=len(payload)
|
||||||
|
)
|
||||||
|
MediaLocation.objects.create(
|
||||||
|
item=self.item, path=str(path), rel_path=path.name, mtime=time.time()
|
||||||
|
)
|
||||||
|
self.client = Client()
|
||||||
|
|
||||||
|
def signed(self, action):
|
||||||
|
return {
|
||||||
|
"sig": sign_payload(
|
||||||
|
{"item": self.item.id, "user": self.user.id, "action": action},
|
||||||
|
services.MEDIA_FILE_SALT,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_media_response_carries_cache_headers(self):
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/files/J-{self.item.id}/raw/", self.signed("raw")
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn("private", response["Cache-Control"])
|
||||||
|
self.assertIn(
|
||||||
|
f"max-age={services.MEDIA_CACHE_SECONDS}", response["Cache-Control"]
|
||||||
|
)
|
||||||
|
self.assertIn("immutable", response["Cache-Control"])
|
||||||
|
self.assertTrue(response["ETag"])
|
||||||
|
self.assertTrue(response["Last-Modified"])
|
||||||
|
|
||||||
|
def test_media_revalidation_returns_304(self):
|
||||||
|
url = f"/api/files/J-{self.item.id}/raw/"
|
||||||
|
first = self.client.get(url, self.signed("raw"))
|
||||||
|
second = self.client.get(
|
||||||
|
url, self.signed("raw"), HTTP_IF_NONE_MATCH=first["ETag"]
|
||||||
|
)
|
||||||
|
self.assertEqual(second.status_code, 304)
|
||||||
|
self.assertEqual(second.content, b"")
|
||||||
|
|
||||||
|
def test_image_thumbnail_is_generated_and_reused(self):
|
||||||
|
url = f"/api/files/J-{self.item.id}/thumbnail/"
|
||||||
|
response = self.client.get(url, self.signed("thumbnail"))
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response["Content-Type"], "image/jpeg")
|
||||||
|
thumb = self._media / "thumbs" / f"{self.item.md5}.jpg"
|
||||||
|
self.assertTrue(thumb.exists())
|
||||||
|
with Image.open(thumb) as image:
|
||||||
|
self.assertLessEqual(max(image.size), 480)
|
||||||
|
before = thumb.stat().st_mtime_ns
|
||||||
|
self.client.get(url, self.signed("thumbnail"))
|
||||||
|
self.assertEqual(thumb.stat().st_mtime_ns, before)
|
||||||
|
|
||||||
|
def test_staged_files_cache_briefly(self):
|
||||||
|
temp = TempUpload.objects.create(
|
||||||
|
user=self.user,
|
||||||
|
file=SimpleUploadedFile("staged.png", TINY_PNG, content_type="image/png"),
|
||||||
|
original_filename="staged.png",
|
||||||
|
md5=hashlib.md5(b"staged").hexdigest(),
|
||||||
|
size=len(TINY_PNG),
|
||||||
|
)
|
||||||
|
signature = sign_payload(
|
||||||
|
{"temp": str(temp.id), "user": self.user.id}, services.UPLOAD_FILE_SALT
|
||||||
|
)
|
||||||
|
response = self.client.get(
|
||||||
|
f"/api/uploads/{temp.id}/file/", {"sig": signature}
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn(
|
||||||
|
f"max-age={services.TEMP_CACHE_SECONDS}", response["Cache-Control"]
|
||||||
|
)
|
||||||
|
self.assertNotIn("immutable", response["Cache-Control"])
|
||||||
@@ -170,12 +170,18 @@ class StagedUploadWorkflowTests(TestCase):
|
|||||||
self.assertEqual(len(body["resolved"]), 2)
|
self.assertEqual(len(body["resolved"]), 2)
|
||||||
self.assertEqual(body["errors"], [])
|
self.assertEqual(body["errors"], [])
|
||||||
|
|
||||||
for temp in (first, second):
|
# Completed uploads are notifications now: the staged rows are gone
|
||||||
temp.refresh_from_db()
|
# and the live feed carries the filename -> J-ID mapping.
|
||||||
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
|
self.assertFalse(
|
||||||
self.assertIsNotNone(temp.library_item_id)
|
TempUpload.objects.filter(pk__in=[first.id, second.id]).exists()
|
||||||
self.assertEqual(temp.library_item.rating, "q")
|
)
|
||||||
self.assertEqual(temp.library_item.uploaded_by_id, self.uploader.id)
|
feed = UploadRun.objects.get(user=self.uploader).recent_completions
|
||||||
|
self.assertEqual(
|
||||||
|
{entry["filename"] for entry in feed}, {"one.png", "two.png"}
|
||||||
|
)
|
||||||
|
for item in MediaItem.objects.all():
|
||||||
|
self.assertEqual(item.rating, "q")
|
||||||
|
self.assertEqual(item.uploaded_by_id, self.uploader.id)
|
||||||
untouched.refresh_from_db()
|
untouched.refresh_from_db()
|
||||||
self.assertEqual(untouched.status, TempUpload.STATUS_PENDING)
|
self.assertEqual(untouched.status, TempUpload.STATUS_PENDING)
|
||||||
|
|
||||||
@@ -231,6 +237,25 @@ class StagedUploadWorkflowTests(TestCase):
|
|||||||
self.assertEqual(response.status_code, 200)
|
self.assertEqual(response.status_code, 200)
|
||||||
return seed
|
return seed
|
||||||
|
|
||||||
|
def test_duplicate_upload_returns_a_completion_without_a_record(self):
|
||||||
|
client = self.api_client(self.uploader)
|
||||||
|
first = self.upload_via_api(client, "same.png")
|
||||||
|
self.assertEqual(first.status_code, 201)
|
||||||
|
# Index the first upload so the second one is byte-identical.
|
||||||
|
seed = TempUpload.objects.get(pk=first.json()["temp_id"])
|
||||||
|
self.resolve_bulk(client, [seed.id], "s")
|
||||||
|
|
||||||
|
response = self.upload_via_api(client, "same.png")
|
||||||
|
self.assertEqual(response.status_code, 201)
|
||||||
|
body = response.json()
|
||||||
|
self.assertEqual(body["status"], TempUpload.STATUS_COMPLETED)
|
||||||
|
self.assertEqual(body["resolution"], TempUpload.RESOLUTION_DUPLICATE)
|
||||||
|
self.assertTrue(body["library_j_id"].startswith("J-"))
|
||||||
|
# Duplicates never become board records; the feed announces them.
|
||||||
|
self.assertFalse(TempUpload.objects.filter(pk=body["temp_id"]).exists())
|
||||||
|
feed = UploadRun.objects.get(user=self.uploader).recent_completions
|
||||||
|
self.assertEqual(feed[-1]["filename"], "same.png")
|
||||||
|
|
||||||
def test_upload_defers_visual_similarity_to_its_phase(self):
|
def test_upload_defers_visual_similarity_to_its_phase(self):
|
||||||
client = self.api_client(self.uploader)
|
client = self.api_client(self.uploader)
|
||||||
self.seed_library_item(client, "seed-defer")
|
self.seed_library_item(client, "seed-defer")
|
||||||
@@ -312,14 +337,23 @@ class StagedUploadWorkflowTests(TestCase):
|
|||||||
body = response.json()
|
body = response.json()
|
||||||
self.assertEqual(body["errors"], [])
|
self.assertEqual(body["errors"], [])
|
||||||
self.assertEqual(len(body["updated"]), 2)
|
self.assertEqual(len(body["updated"]), 2)
|
||||||
for temp, post_id in ((first, 900001), (second, 900002)):
|
for entry, post_id in zip(body["updated"], (900001, 900002)):
|
||||||
temp.refresh_from_db()
|
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
|
||||||
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
|
self.assertEqual(entry["e621_post_id"], post_id)
|
||||||
self.assertEqual(temp.library_item_id is not None, True)
|
self.assertTrue(entry["library_j_id"].startswith("J-"))
|
||||||
self.assertEqual(temp.e621_post_id, post_id)
|
# Indexed uploads no longer leave a board record; the completion feed
|
||||||
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
|
# carries them for the live page instead.
|
||||||
self.assertEqual(temp.library_item.e621_post_id, post_id)
|
self.assertFalse(TempUpload.objects.exists())
|
||||||
self.assertEqual(MediaItem.objects.count(), 2)
|
self.assertEqual(
|
||||||
|
{item.e621_post_id for item in MediaItem.objects.all()},
|
||||||
|
{900001, 900002},
|
||||||
|
)
|
||||||
|
feed = UploadRun.objects.get(user=self.uploader).recent_completions
|
||||||
|
self.assertEqual(len(feed), 2)
|
||||||
|
self.assertEqual(
|
||||||
|
{entry["filename"] for entry in feed},
|
||||||
|
{"bulk-link-1.png", "bulk-link-2.png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class IqdbRecordingTests(TestCase):
|
class IqdbRecordingTests(TestCase):
|
||||||
@@ -461,6 +495,7 @@ class UploadPipelineTests(TestCase):
|
|||||||
|
|
||||||
def test_md5_match_auto_imports_the_file(self):
|
def test_md5_match_auto_imports_the_file(self):
|
||||||
temp = self.stage(label="match")
|
temp = self.stage(label="match")
|
||||||
|
temp_id = temp.id
|
||||||
post = {
|
post = {
|
||||||
"id": 123456,
|
"id": 123456,
|
||||||
"rating": "s",
|
"rating": "s",
|
||||||
@@ -477,17 +512,42 @@ class UploadPipelineTests(TestCase):
|
|||||||
):
|
):
|
||||||
upload_pipeline.run_pipeline(self.uploader.id)
|
upload_pipeline.run_pipeline(self.uploader.id)
|
||||||
|
|
||||||
temp.refresh_from_db()
|
# The indexed upload leaves no board record; the feed reports it.
|
||||||
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
|
self.assertFalse(TempUpload.objects.filter(pk=temp_id).exists())
|
||||||
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
|
item = MediaItem.objects.get(e621_post_id=123456)
|
||||||
self.assertEqual(temp.e621_post_id, 123456)
|
self.assertEqual(item.uploaded_by_id, self.uploader.id)
|
||||||
self.assertIsNotNone(temp.library_item_id)
|
|
||||||
self.assertIsNotNone(temp.e621_checked_at)
|
|
||||||
self.assertIsNone(temp.claimed_at)
|
|
||||||
run = UploadRun.objects.get(user=self.uploader)
|
run = UploadRun.objects.get(user=self.uploader)
|
||||||
self.assertEqual(run.status, UploadRun.STATUS_IDLE)
|
self.assertEqual(run.status, UploadRun.STATUS_IDLE)
|
||||||
self.assertEqual(run.matched, 1)
|
self.assertEqual(run.matched, 1)
|
||||||
self.assertEqual(run.processed, 1)
|
self.assertEqual(run.processed, 1)
|
||||||
|
self.assertEqual(len(run.recent_completions), 1)
|
||||||
|
entry = run.recent_completions[0]
|
||||||
|
self.assertEqual(entry["id"], str(temp_id))
|
||||||
|
self.assertEqual(entry["item_id"], item.id)
|
||||||
|
self.assertEqual(entry["filename"], "match.png")
|
||||||
|
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
|
||||||
|
|
||||||
|
def test_status_reports_the_completion_feed(self):
|
||||||
|
temp = self.stage(label="feed")
|
||||||
|
client = self.api_client(self.uploader)
|
||||||
|
post = {
|
||||||
|
"id": 654321,
|
||||||
|
"rating": "s",
|
||||||
|
"file": {"md5": temp.md5, "url": "https://static1.e621.net/data/f.png"},
|
||||||
|
}
|
||||||
|
with mock.patch.object(
|
||||||
|
upload_pipeline.e621,
|
||||||
|
"check_md5_batch",
|
||||||
|
return_value={temp.md5: post},
|
||||||
|
):
|
||||||
|
upload_pipeline.run_pipeline(self.uploader.id)
|
||||||
|
|
||||||
|
body = client.get("/api/uploads/status/").json()
|
||||||
|
completions = body["recent_completions"]
|
||||||
|
self.assertEqual(len(completions), 1)
|
||||||
|
self.assertEqual(completions[0]["filename"], "feed.png")
|
||||||
|
self.assertEqual(completions[0]["j_id"], f"J-{MediaItem.objects.get().id}")
|
||||||
|
self.assertIn("/thumbnail/", completions[0]["thumbnail_url"])
|
||||||
|
|
||||||
def test_unmatched_file_runs_every_phase(self):
|
def test_unmatched_file_runs_every_phase(self):
|
||||||
temp = self.stage(label="nomatch")
|
temp = self.stage(label="nomatch")
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ from django.db.models import F, Q
|
|||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from . import e621, services
|
from . import e621, services
|
||||||
from .models import TempUpload, UploadRun
|
from .models import MediaItem, TempUpload, UploadRun
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -46,6 +46,9 @@ MAX_ATTEMPTS = 3
|
|||||||
# Round-level e621 retries before the run is paused.
|
# Round-level e621 retries before the run is paused.
|
||||||
ROUND_ATTEMPTS = 3
|
ROUND_ATTEMPTS = 3
|
||||||
ROUND_RETRY_SECONDS = 20
|
ROUND_RETRY_SECONDS = 20
|
||||||
|
# Completions kept in the live feed. The board only shows what happened while
|
||||||
|
# the page was open, so a bounded rolling window is plenty.
|
||||||
|
COMPLETION_FEED_LIMIT = 200
|
||||||
|
|
||||||
WORK_STATUSES = (TempUpload.STATUS_PENDING, TempUpload.STATUS_VISUAL_MATCH)
|
WORK_STATUSES = (TempUpload.STATUS_PENDING, TempUpload.STATUS_VISUAL_MATCH)
|
||||||
VIDEO_RE = r"\.(mp4|webm)$"
|
VIDEO_RE = r"\.(mp4|webm)$"
|
||||||
@@ -116,7 +119,7 @@ def waiting_counts(user):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def status_payload(user):
|
def status_payload(user, request=None):
|
||||||
"""Cheap state for the shell/upload page to poll."""
|
"""Cheap state for the shell/upload page to poll."""
|
||||||
run = UploadRun.objects.filter(user=user).first()
|
run = UploadRun.objects.filter(user=user).first()
|
||||||
outstanding = count_outstanding(user)
|
outstanding = count_outstanding(user)
|
||||||
@@ -141,10 +144,67 @@ def status_payload(user):
|
|||||||
"error": run.error if run is not None else "",
|
"error": run.error if run is not None else "",
|
||||||
"outstanding": outstanding,
|
"outstanding": outstanding,
|
||||||
"waiting": waiting_counts(user),
|
"waiting": waiting_counts(user),
|
||||||
|
"recent_completions": completion_payload(run, user, request=request),
|
||||||
"updated_at": run.updated_at.isoformat() if run is not None else None,
|
"updated_at": run.updated_at.isoformat() if run is not None else None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def completion_payload(run, user, request=None):
|
||||||
|
"""The live completion feed: filename -> J-ID for freshly indexed uploads.
|
||||||
|
|
||||||
|
Completed ``TempUpload`` rows are deleted, so this is the only place the
|
||||||
|
board learns about them. It is a notification feed, not durable state:
|
||||||
|
bounded, never dismissed, and ignored by fresh page loads.
|
||||||
|
"""
|
||||||
|
entries = list(run.recent_completions or []) if run is not None else []
|
||||||
|
if not entries:
|
||||||
|
return []
|
||||||
|
ids = [entry.get("item_id") for entry in entries if entry.get("item_id")]
|
||||||
|
items = MediaItem.objects.in_bulk(ids)
|
||||||
|
out = []
|
||||||
|
for entry in reversed(entries): # newest first
|
||||||
|
item = items.get(entry.get("item_id"))
|
||||||
|
if item is None:
|
||||||
|
continue
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"id": entry.get("id"),
|
||||||
|
"filename": entry.get("filename"),
|
||||||
|
"j_id": f"J-{item.id}",
|
||||||
|
"resolution": entry.get("resolution", ""),
|
||||||
|
"post_id": entry.get("post_id"),
|
||||||
|
"thumbnail_url": services.signed_media_url(
|
||||||
|
item, user, "thumbnail", request=request
|
||||||
|
),
|
||||||
|
"at": entry.get("at"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def record_completion(temp, item, resolution=""):
|
||||||
|
"""Append one completion to the owner's feed; never fails an import."""
|
||||||
|
entry = {
|
||||||
|
"id": str(temp.pk),
|
||||||
|
"filename": temp.original_filename,
|
||||||
|
"item_id": item.pk,
|
||||||
|
"resolution": resolution or temp.resolution or "",
|
||||||
|
"post_id": temp.e621_post_id,
|
||||||
|
"at": timezone.now().isoformat(),
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
with transaction.atomic():
|
||||||
|
run, _ = UploadRun.objects.select_for_update().get_or_create(
|
||||||
|
user_id=temp.user_id
|
||||||
|
)
|
||||||
|
feed = list(run.recent_completions or [])
|
||||||
|
feed.append(entry)
|
||||||
|
run.recent_completions = feed[-COMPLETION_FEED_LIMIT:]
|
||||||
|
run.save(update_fields=["recent_completions", "updated_at"])
|
||||||
|
except Exception: # noqa: BLE001 - a notification must not break an import
|
||||||
|
logger.exception("Could not record the completion of %s", temp.pk)
|
||||||
|
|
||||||
|
|
||||||
def reap_stale_claims():
|
def reap_stale_claims():
|
||||||
"""Queue rows left claimed by a recycled worker and pause dead runs."""
|
"""Queue rows left claimed by a recycled worker and pause dead runs."""
|
||||||
cutoff = timezone.now() - STALE_CLAIM_AFTER
|
cutoff = timezone.now() - STALE_CLAIM_AFTER
|
||||||
@@ -237,20 +297,17 @@ def run_pipeline(user_id):
|
|||||||
break
|
break
|
||||||
process_round(run, user, rows)
|
process_round(run, user, rows)
|
||||||
except PipelinePaused as exc:
|
except PipelinePaused as exc:
|
||||||
run.status = UploadRun.STATUS_PAUSED
|
_save_run(run, status=UploadRun.STATUS_PAUSED, phase="", error=str(exc))
|
||||||
run.phase = ""
|
|
||||||
run.error = str(exc)
|
|
||||||
run.save()
|
|
||||||
except Exception as exc: # noqa: BLE001 - surface crashes as a run error
|
except Exception as exc: # noqa: BLE001 - surface crashes as a run error
|
||||||
logger.exception("Upload pipeline for user %s failed", user_id)
|
logger.exception("Upload pipeline for user %s failed", user_id)
|
||||||
run.status = UploadRun.STATUS_ERROR
|
_save_run(
|
||||||
run.phase = ""
|
run,
|
||||||
run.error = f"The upload pipeline stopped: {exc}"
|
status=UploadRun.STATUS_ERROR,
|
||||||
run.save()
|
phase="",
|
||||||
|
error=f"The upload pipeline stopped: {exc}",
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
run.status = UploadRun.STATUS_IDLE
|
_save_run(run, status=UploadRun.STATUS_IDLE, phase="")
|
||||||
run.phase = ""
|
|
||||||
run.save()
|
|
||||||
|
|
||||||
|
|
||||||
def claim_round(user, size=CLAIM_SIZE):
|
def claim_round(user, size=CLAIM_SIZE):
|
||||||
@@ -460,7 +517,9 @@ def finalize_round(run, ids, matched):
|
|||||||
TempUpload.objects.filter(pk__in=release).update(claimed_at=None)
|
TempUpload.objects.filter(pk__in=release).update(claimed_at=None)
|
||||||
_save_run(
|
_save_run(
|
||||||
run,
|
run,
|
||||||
processed=run.processed + len(finished),
|
# Completed rows are deleted as they are imported, so they cannot be
|
||||||
|
# seen in the refreshed rows; count the matches explicitly.
|
||||||
|
processed=run.processed + len(finished) + matched,
|
||||||
failed=run.failed + len(failed - finished),
|
failed=run.failed + len(failed - finished),
|
||||||
matched=run.matched + matched,
|
matched=run.matched + matched,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ from urllib.parse import urlparse
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
from django.core import signing
|
|
||||||
from django.core.exceptions import ValidationError
|
from django.core.exceptions import ValidationError
|
||||||
from django.http import Http404
|
from django.http import Http404
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
@@ -30,6 +29,7 @@ from . import services
|
|||||||
from .models import MediaItem, TempUpload
|
from .models import MediaItem, TempUpload
|
||||||
from .permissions import CanUpload
|
from .permissions import CanUpload
|
||||||
from .serializers import TempUploadListSerializer, TempUploadSerializer
|
from .serializers import TempUploadListSerializer, TempUploadSerializer
|
||||||
|
from .signing_urls import load_payload
|
||||||
from .tools import HASH_FIELDS, hashed_items, hashes_similarity
|
from .tools import HASH_FIELDS, hashed_items, hashes_similarity
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -158,6 +158,13 @@ def complete_temp_upload(temp, download_url=None):
|
|||||||
item.save(update_fields=update_fields + ["updated_at"])
|
item.save(update_fields=update_fields + ["updated_at"])
|
||||||
|
|
||||||
temp.save()
|
temp.save()
|
||||||
|
from .upload_pipeline import record_completion
|
||||||
|
|
||||||
|
record_completion(temp, item)
|
||||||
|
# The board learns about completions from the live feed, so the record is
|
||||||
|
# deleted as soon as the file is indexed: nothing left to dismiss. Delete
|
||||||
|
# through the queryset so callers keep ``temp.pk`` for their response.
|
||||||
|
TempUpload.objects.filter(pk=temp.pk).delete()
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
||||||
@@ -190,6 +197,28 @@ class TempUploadViewSet(
|
|||||||
user=self.request.user
|
user=self.request.user
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _completed_payload(self, temp, item):
|
||||||
|
"""Synthetic row for an upload that is indexed immediately.
|
||||||
|
|
||||||
|
Duplicates and resolved uploads never leave a board record; the SPA
|
||||||
|
turns this response (or the live completion feed) into a "J-x
|
||||||
|
uploaded" card that lives only in the page session.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"temp_id": str(temp.pk),
|
||||||
|
"original_filename": temp.original_filename,
|
||||||
|
"md5": temp.md5,
|
||||||
|
"size": temp.size,
|
||||||
|
"status": TempUpload.STATUS_COMPLETED,
|
||||||
|
"resolution": temp.resolution,
|
||||||
|
"e621_post_id": temp.e621_post_id,
|
||||||
|
"library_j_id": f"J-{item.id}",
|
||||||
|
"file_url": None,
|
||||||
|
"preview_url": services.signed_media_url(
|
||||||
|
item, self.request.user, "thumbnail", request=self.request
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
def create(self, request):
|
def create(self, request):
|
||||||
upload = request.FILES.get("file")
|
upload = request.FILES.get("file")
|
||||||
if upload is None:
|
if upload is None:
|
||||||
@@ -217,6 +246,13 @@ class TempUploadViewSet(
|
|||||||
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
|
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
|
||||||
temp.library_item = existing
|
temp.library_item = existing
|
||||||
temp.file.delete(save=False)
|
temp.file.delete(save=False)
|
||||||
|
temp.save()
|
||||||
|
from .upload_pipeline import record_completion
|
||||||
|
|
||||||
|
record_completion(temp, existing)
|
||||||
|
payload = self._completed_payload(temp, existing)
|
||||||
|
TempUpload.objects.filter(pk=temp.pk).delete()
|
||||||
|
return Response(payload, status=status.HTTP_201_CREATED)
|
||||||
# Visual similarity and IQDB run in the background pipeline so a large
|
# Visual similarity and IQDB run in the background pipeline so a large
|
||||||
# batch uploads at full speed and the work survives the browser.
|
# batch uploads at full speed and the work survives the browser.
|
||||||
temp.save()
|
temp.save()
|
||||||
@@ -257,7 +293,7 @@ class TempUploadViewSet(
|
|||||||
"""Cheap pipeline state for the shell indicator and the upload page."""
|
"""Cheap pipeline state for the shell indicator and the upload page."""
|
||||||
from .upload_pipeline import status_payload
|
from .upload_pipeline import status_payload
|
||||||
|
|
||||||
return Response(status_payload(request.user))
|
return Response(status_payload(request.user, request=request))
|
||||||
|
|
||||||
@action(detail=False, methods=["post"])
|
@action(detail=False, methods=["post"])
|
||||||
def process(self, request):
|
def process(self, request):
|
||||||
@@ -269,7 +305,7 @@ class TempUploadViewSet(
|
|||||||
from .upload_pipeline import start_pipeline, status_payload
|
from .upload_pipeline import start_pipeline, status_payload
|
||||||
|
|
||||||
start_pipeline(request.user)
|
start_pipeline(request.user)
|
||||||
return Response(status_payload(request.user))
|
return Response(status_payload(request.user, request=request))
|
||||||
|
|
||||||
@action(detail=True, methods=["post"])
|
@action(detail=True, methods=["post"])
|
||||||
def retry(self, request, pk=None):
|
def retry(self, request, pk=None):
|
||||||
@@ -314,7 +350,7 @@ class TempUploadViewSet(
|
|||||||
update["status"] = TempUpload.STATUS_PENDING
|
update["status"] = TempUpload.STATUS_PENDING
|
||||||
TempUpload.objects.filter(pk=temp.pk).update(**update)
|
TempUpload.objects.filter(pk=temp.pk).update(**update)
|
||||||
start_pipeline(request.user)
|
start_pipeline(request.user)
|
||||||
return Response(status_payload(request.user))
|
return Response(status_payload(request.user, request=request))
|
||||||
|
|
||||||
@action(detail=False, methods=["post"], url_path="retry-all")
|
@action(detail=False, methods=["post"], url_path="retry-all")
|
||||||
def retry_all(self, request):
|
def retry_all(self, request):
|
||||||
@@ -340,7 +376,7 @@ class TempUploadViewSet(
|
|||||||
e621_checked_at=None,
|
e621_checked_at=None,
|
||||||
)
|
)
|
||||||
start_pipeline(request.user)
|
start_pipeline(request.user)
|
||||||
return Response(status_payload(request.user))
|
return Response(status_payload(request.user, request=request))
|
||||||
|
|
||||||
@action(detail=False, methods=["post"], url_path="discard-bulk")
|
@action(detail=False, methods=["post"], url_path="discard-bulk")
|
||||||
def discard_bulk(self, request):
|
def discard_bulk(self, request):
|
||||||
@@ -393,14 +429,7 @@ class TempUploadViewSet(
|
|||||||
if user is None:
|
if user is None:
|
||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
if signature:
|
if signature:
|
||||||
try:
|
payload = load_payload(signature, services.UPLOAD_FILE_SALT)
|
||||||
payload = signing.loads(
|
|
||||||
signature,
|
|
||||||
salt=services.UPLOAD_FILE_SALT,
|
|
||||||
max_age=86400,
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
payload = None
|
|
||||||
if payload and str(payload.get("temp")) == str(pk):
|
if payload and str(payload.get("temp")) == str(pk):
|
||||||
user = (
|
user = (
|
||||||
get_user_model()
|
get_user_model()
|
||||||
@@ -501,7 +530,7 @@ class TempUploadViewSet(
|
|||||||
|
|
||||||
temp.save()
|
temp.save()
|
||||||
try:
|
try:
|
||||||
complete_temp_upload(temp, download_url=download_url)
|
item = complete_temp_upload(temp, download_url=download_url)
|
||||||
except Exception as exc: # noqa: BLE001 - report completion failures
|
except Exception as exc: # noqa: BLE001 - report completion failures
|
||||||
logger.exception("Could not complete staged upload %s", temp.id)
|
logger.exception("Could not complete staged upload %s", temp.id)
|
||||||
temp.status = TempUpload.STATUS_ERROR
|
temp.status = TempUpload.STATUS_ERROR
|
||||||
@@ -510,8 +539,7 @@ class TempUploadViewSet(
|
|||||||
{"detail": f"Could not finish the upload: {exc}"},
|
{"detail": f"Could not finish the upload: {exc}"},
|
||||||
status=status.HTTP_400_BAD_REQUEST,
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
)
|
)
|
||||||
temp.refresh_from_db()
|
return Response(self._completed_payload(temp, item))
|
||||||
return Response(self.get_serializer(temp).data)
|
|
||||||
|
|
||||||
@action(detail=False, methods=["post"], url_path="link-bulk")
|
@action(detail=False, methods=["post"], url_path="link-bulk")
|
||||||
def link_bulk(self, request):
|
def link_bulk(self, request):
|
||||||
@@ -583,15 +611,14 @@ class TempUploadViewSet(
|
|||||||
candidate_url = ""
|
candidate_url = ""
|
||||||
temp.save()
|
temp.save()
|
||||||
try:
|
try:
|
||||||
complete_temp_upload(temp, download_url=candidate_url or None)
|
item = complete_temp_upload(temp, download_url=candidate_url or None)
|
||||||
except Exception as exc: # noqa: BLE001 - report per-file failures
|
except Exception as exc: # noqa: BLE001 - report per-file failures
|
||||||
logger.exception("Could not complete staged upload %s", temp.id)
|
logger.exception("Could not complete staged upload %s", temp.id)
|
||||||
temp.status = TempUpload.STATUS_ERROR
|
temp.status = TempUpload.STATUS_ERROR
|
||||||
temp.save(update_fields=["status", "updated_at"])
|
temp.save(update_fields=["status", "updated_at"])
|
||||||
errors.append({"temp_id": temp_id, "error": str(exc)})
|
errors.append({"temp_id": temp_id, "error": str(exc)})
|
||||||
continue
|
continue
|
||||||
temp.refresh_from_db()
|
updated.append(self._completed_payload(temp, item))
|
||||||
updated.append(self.get_serializer(temp).data)
|
|
||||||
|
|
||||||
return Response({"updated": updated, "errors": errors})
|
return Response({"updated": updated, "errors": errors})
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from pathlib import Path
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
|
||||||
from django.db.models import Min, Q
|
from django.db.models import Min, Q
|
||||||
from django.http import Http404, StreamingHttpResponse
|
from django.http import Http404, StreamingHttpResponse
|
||||||
from django.shortcuts import get_object_or_404
|
from django.shortcuts import get_object_or_404
|
||||||
@@ -31,6 +30,7 @@ from .serializers import (
|
|||||||
MatchTaskSerializer,
|
MatchTaskSerializer,
|
||||||
MediaItemSerializer,
|
MediaItemSerializer,
|
||||||
)
|
)
|
||||||
|
from .signing_urls import load_payload
|
||||||
|
|
||||||
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
||||||
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
||||||
@@ -132,11 +132,8 @@ class MediaItemViewSet(
|
|||||||
signature = request.query_params.get("sig")
|
signature = request.query_params.get("sig")
|
||||||
if not signature:
|
if not signature:
|
||||||
return None
|
return None
|
||||||
try:
|
payload = load_payload(signature, services.MEDIA_FILE_SALT)
|
||||||
payload = signing.loads(
|
if payload is None:
|
||||||
signature, salt=services.MEDIA_FILE_SALT, max_age=86400
|
|
||||||
)
|
|
||||||
except signing.BadSignature:
|
|
||||||
return None
|
return None
|
||||||
if payload.get("action") != action_name:
|
if payload.get("action") != action_name:
|
||||||
return None
|
return None
|
||||||
@@ -158,7 +155,11 @@ class MediaItemViewSet(
|
|||||||
status=status.HTTP_404_NOT_FOUND,
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
)
|
)
|
||||||
return services.serve_file(
|
return services.serve_file(
|
||||||
request, location.path, download=request.query_params.get("download") == "1"
|
request,
|
||||||
|
location.path,
|
||||||
|
download=request.query_params.get("download") == "1",
|
||||||
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
@action(detail=True, methods=["get"], throttle_classes=[])
|
@action(detail=True, methods=["get"], throttle_classes=[])
|
||||||
@@ -173,13 +174,26 @@ class MediaItemViewSet(
|
|||||||
path = Path(location.path)
|
path = Path(location.path)
|
||||||
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
||||||
thumbnail = services.generate_video_thumbnail(item.md5, path)
|
thumbnail = services.generate_video_thumbnail(item.md5, path)
|
||||||
if thumbnail is None:
|
else:
|
||||||
return Response(
|
thumbnail = services.generate_image_thumbnail(item.md5, path)
|
||||||
{"detail": "Thumbnail unavailable."},
|
if thumbnail is not None:
|
||||||
status=status.HTTP_404_NOT_FOUND,
|
return services.serve_file(
|
||||||
)
|
request,
|
||||||
return services.serve_file(request, thumbnail)
|
thumbnail,
|
||||||
return services.serve_file(request, path)
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
|
)
|
||||||
|
if path.suffix.lower() in services.VIDEO_EXTENSIONS:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Thumbnail unavailable."},
|
||||||
|
status=status.HTTP_404_NOT_FOUND,
|
||||||
|
)
|
||||||
|
return services.serve_file(
|
||||||
|
request,
|
||||||
|
path,
|
||||||
|
max_age=services.MEDIA_CACHE_SECONDS,
|
||||||
|
immutable=True,
|
||||||
|
)
|
||||||
|
|
||||||
@action(detail=False, methods=["post"], permission_classes=[AllowAny])
|
@action(detail=False, methods=["post"], permission_classes=[AllowAny])
|
||||||
def lookup(self, request):
|
def lookup(self, request):
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ const ratingLabels: Record<string, string> = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
export function MediaCard({ item }: { item: MediaItem }) {
|
export function MediaCard({ item }: { item: MediaItem }) {
|
||||||
const preview = apiUrl(
|
// The thumbnail endpoint now builds real 480px previews for images too, so
|
||||||
item.kind === "video" ? item.thumbnail_url : item.raw_url,
|
// the grid no longer pulls full-size originals.
|
||||||
);
|
const preview = apiUrl(item.thumbnail_url);
|
||||||
const rating = item.display_rating;
|
const rating = item.display_rating;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|||||||
@@ -28,7 +28,13 @@ import {
|
|||||||
RATING_LABELS,
|
RATING_LABELS,
|
||||||
} from "@/lib/e621";
|
} from "@/lib/e621";
|
||||||
import { formatBytes } from "@/lib/format";
|
import { formatBytes } from "@/lib/format";
|
||||||
import type { E621IqdbCandidate, Rating, TempUpload } from "@/lib/types";
|
import type {
|
||||||
|
E621IqdbCandidate,
|
||||||
|
Rating,
|
||||||
|
TempUpload,
|
||||||
|
UploadCompletion,
|
||||||
|
UploadStagingResult,
|
||||||
|
} from "@/lib/types";
|
||||||
import {
|
import {
|
||||||
postProcessUploads,
|
postProcessUploads,
|
||||||
uploadStatusQueryKey,
|
uploadStatusQueryKey,
|
||||||
@@ -272,6 +278,21 @@ function TempCard({
|
|||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
{temp.status === "error" ? (
|
||||||
|
<div className="mt-2 flex flex-wrap items-center gap-1.5">
|
||||||
|
<Button className="px-2 py-1 text-xs" onClick={onRetry}>
|
||||||
|
Retry
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
variant="secondary"
|
||||||
|
className="px-2 py-1 text-xs"
|
||||||
|
onClick={onOpen}
|
||||||
|
>
|
||||||
|
Set metadata
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={onDismiss}
|
onClick={onDismiss}
|
||||||
@@ -289,6 +310,58 @@ function TempCard({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A "J-x uploaded" notification.
|
||||||
|
*
|
||||||
|
* Completions are not stored on the board: the pipeline deletes the staged
|
||||||
|
* record and announces it through the status feed, so these cards only exist
|
||||||
|
* while the page is open.
|
||||||
|
*/
|
||||||
|
function CompletionCard({ completion }: { completion: UploadCompletion }) {
|
||||||
|
const preview = apiUrl(completion.thumbnail_url ?? "");
|
||||||
|
return (
|
||||||
|
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-2.5">
|
||||||
|
<div className="relative aspect-video overflow-hidden rounded-md bg-ctp-mantle">
|
||||||
|
{preview ? (
|
||||||
|
<img
|
||||||
|
src={preview}
|
||||||
|
alt={completion.filename}
|
||||||
|
loading="lazy"
|
||||||
|
className="h-full w-full object-cover"
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="flex h-full w-full items-center justify-center text-xs text-ctp-overlay0">
|
||||||
|
No preview
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<span className="absolute right-2 top-2 flex items-center gap-1 rounded-full bg-ctp-green/90 px-1.5 py-0.5 font-mono text-[10px] font-medium text-ctp-crust">
|
||||||
|
<CheckCircle2 className="h-2.5 w-2.5" />
|
||||||
|
indexed
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p
|
||||||
|
className="mt-2 truncate font-mono text-xs text-ctp-subtext1"
|
||||||
|
title={completion.filename}
|
||||||
|
>
|
||||||
|
{completion.filename}
|
||||||
|
</p>
|
||||||
|
<p className="mt-1 flex items-center gap-1.5 text-xs text-ctp-green">
|
||||||
|
<CheckCircle2 className="h-3.5 w-3.5 shrink-0" />
|
||||||
|
<Link
|
||||||
|
to={`/detail/${completion.j_id}`}
|
||||||
|
className="font-mono hover:underline"
|
||||||
|
>
|
||||||
|
{completion.j_id}
|
||||||
|
</Link>
|
||||||
|
<span className="truncate text-ctp-overlay0">
|
||||||
|
{RESOLUTION_LABELS[completion.resolution] ?? completion.resolution}
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Full metadata editor for one staged upload.
|
* Full metadata editor for one staged upload.
|
||||||
*
|
*
|
||||||
@@ -816,6 +889,10 @@ export default function UploadPage() {
|
|||||||
const [discarding, setDiscarding] = useState(false);
|
const [discarding, setDiscarding] = useState(false);
|
||||||
const [modalId, setModalId] = useState<string | null>(null);
|
const [modalId, setModalId] = useState<string | null>(null);
|
||||||
const [bulkOpen, setBulkOpen] = useState(false);
|
const [bulkOpen, setBulkOpen] = useState(false);
|
||||||
|
// Indexed uploads shown while this page is open (never persisted).
|
||||||
|
const [completions, setCompletions] = useState<UploadCompletion[]>([]);
|
||||||
|
const completionsSeeded = useRef(false);
|
||||||
|
const seenCompletions = useRef<Set<string>>(new Set());
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const urls = objectUrls.current;
|
const urls = objectUrls.current;
|
||||||
@@ -836,6 +913,27 @@ export default function UploadPage() {
|
|||||||
});
|
});
|
||||||
const uploads = uploadsQuery.data ?? [];
|
const uploads = uploadsQuery.data ?? [];
|
||||||
|
|
||||||
|
// Completions are session notifications. Seed what the server already has
|
||||||
|
// when the page opens (so a reload does not resurrect old ones) and append
|
||||||
|
// only what happens while the page is watching.
|
||||||
|
useEffect(() => {
|
||||||
|
const feed = status?.recent_completions;
|
||||||
|
if (!feed) return;
|
||||||
|
if (!completionsSeeded.current) {
|
||||||
|
completionsSeeded.current = true;
|
||||||
|
// Merge: a staging duplicate may have been added before the first poll.
|
||||||
|
seenCompletions.current = new Set([
|
||||||
|
...seenCompletions.current,
|
||||||
|
...feed.map((entry) => entry.id),
|
||||||
|
]);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const fresh = feed.filter((entry) => !seenCompletions.current.has(entry.id));
|
||||||
|
if (fresh.length === 0) return;
|
||||||
|
for (const entry of fresh) seenCompletions.current.add(entry.id);
|
||||||
|
setCompletions((current) => [...fresh, ...current].slice(0, 200));
|
||||||
|
}, [status]);
|
||||||
|
|
||||||
function cacheStatus(data: unknown) {
|
function cacheStatus(data: unknown) {
|
||||||
queryClient.setQueryData(uploadStatusQueryKey, data);
|
queryClient.setQueryData(uploadStatusQueryKey, data);
|
||||||
}
|
}
|
||||||
@@ -892,9 +990,7 @@ export default function UploadPage() {
|
|||||||
|
|
||||||
const pending = uploads.filter((temp) => temp.status === "pending");
|
const pending = uploads.filter((temp) => temp.status === "pending");
|
||||||
const visual = uploads.filter((temp) => temp.status === "visual_match");
|
const visual = uploads.filter((temp) => temp.status === "visual_match");
|
||||||
const completed = uploads.filter(
|
const failed = uploads.filter((temp) => temp.status === "error");
|
||||||
(temp) => temp.status === "completed" || temp.status === "error",
|
|
||||||
);
|
|
||||||
|
|
||||||
// Active work stays in insertion order (the grid must not reshuffle while
|
// Active work stays in insertion order (the grid must not reshuffle while
|
||||||
// you scroll it); the in-flight files get their own strip above it.
|
// you scroll it); the in-flight files get their own strip above it.
|
||||||
@@ -1001,10 +1097,26 @@ export default function UploadPage() {
|
|||||||
error: undefined,
|
error: undefined,
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
await uploadFile(entry.file, {}, (percent) =>
|
const result = await uploadFile<UploadStagingResult>(
|
||||||
reportProgress(entry.id, percent),
|
entry.file,
|
||||||
|
{},
|
||||||
|
(percent) => reportProgress(entry.id, percent),
|
||||||
);
|
);
|
||||||
finishEntry(entry);
|
finishEntry(entry);
|
||||||
|
// An exact duplicate completes during staging: show its card now.
|
||||||
|
if (result?.status === "completed" && result.library_j_id) {
|
||||||
|
const completion: UploadCompletion = {
|
||||||
|
id: result.temp_id,
|
||||||
|
filename: result.original_filename,
|
||||||
|
j_id: result.library_j_id,
|
||||||
|
resolution: result.resolution,
|
||||||
|
post_id: result.e621_post_id ?? null,
|
||||||
|
thumbnail_url: result.preview_url ?? null,
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
seenCompletions.current.add(completion.id);
|
||||||
|
setCompletions((current) => [completion, ...current].slice(0, 200));
|
||||||
|
}
|
||||||
// Processing starts while the rest of the drop is still uploading.
|
// Processing starts while the rest of the drop is still uploading.
|
||||||
kickPipeline();
|
kickPipeline();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -1060,39 +1172,40 @@ export default function UploadPage() {
|
|||||||
.catch((error) => toast.error(errorMessage(error)));
|
.catch((error) => toast.error(errorMessage(error)));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function discardAll(
|
async function discardAll(items: TempUpload[]) {
|
||||||
items: TempUpload[],
|
|
||||||
kind: "discard" | "dismiss",
|
|
||||||
) {
|
|
||||||
if (items.length === 0 || discarding) return;
|
if (items.length === 0 || discarding) return;
|
||||||
const noun = kind === "discard" ? "Discard" : "Dismiss";
|
|
||||||
const confirmed = await confirmAction({
|
const confirmed = await confirmAction({
|
||||||
title: `${noun} ${items.length} record${items.length === 1 ? "" : "s"}?`,
|
title: `Discard ${items.length} record${items.length === 1 ? "" : "s"}?`,
|
||||||
description:
|
description:
|
||||||
kind === "discard"
|
"The staged files are deleted from the staging folder and never enter the library. Files the pipeline is importing right now are skipped.",
|
||||||
? "The staged files are deleted from the staging folder and never enter the library. Files the pipeline is importing right now are skipped."
|
confirmLabel: "Discard all",
|
||||||
: "This only removes the board records; library files are untouched.",
|
danger: true,
|
||||||
confirmLabel: `${noun} all`,
|
|
||||||
danger: kind === "discard",
|
|
||||||
});
|
});
|
||||||
if (!confirmed) return;
|
if (!confirmed) return;
|
||||||
setDiscarding(true);
|
setDiscarding(true);
|
||||||
try {
|
try {
|
||||||
const result = await api<{
|
// The endpoint caps one request at 1000 ids; large staging backlogs
|
||||||
discarded: string[];
|
// must still be discardable in one go.
|
||||||
errors: { temp_id: string; error: string }[];
|
const ids = items.map((temp) => temp.temp_id);
|
||||||
}>("/api/uploads/discard-bulk/", {
|
const chunkSize = 500;
|
||||||
method: "POST",
|
const discarded: string[] = [];
|
||||||
json: { temp_ids: items.map((temp) => temp.temp_id) },
|
const errors: { temp_id: string; error: string }[] = [];
|
||||||
});
|
for (let start = 0; start < ids.length; start += chunkSize) {
|
||||||
const verb = kind === "discard" ? "Discarded" : "Dismissed";
|
const result = await api<{
|
||||||
if (result.discarded.length > 0) {
|
discarded: string[];
|
||||||
toast.ok(`${verb} ${result.discarded.length} record(s).`);
|
errors: { temp_id: string; error: string }[];
|
||||||
|
}>("/api/uploads/discard-bulk/", {
|
||||||
|
method: "POST",
|
||||||
|
json: { temp_ids: ids.slice(start, start + chunkSize) },
|
||||||
|
});
|
||||||
|
discarded.push(...result.discarded);
|
||||||
|
errors.push(...result.errors);
|
||||||
}
|
}
|
||||||
if (result.errors.length > 0) {
|
if (discarded.length > 0) {
|
||||||
toast.error(
|
toast.ok(`Discarded ${discarded.length} record(s).`);
|
||||||
`${result.errors.length} record(s) could not be removed.`,
|
}
|
||||||
);
|
if (errors.length > 0) {
|
||||||
|
toast.error(`${errors.length} record(s) could not be removed.`);
|
||||||
}
|
}
|
||||||
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
|
void queryClient.invalidateQueries({ queryKey: ["uploads"] });
|
||||||
void queryClient.invalidateQueries({ queryKey: ["upload-status"] });
|
void queryClient.invalidateQueries({ queryKey: ["upload-status"] });
|
||||||
@@ -1127,10 +1240,11 @@ export default function UploadPage() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const column = (
|
const column = <T,>(
|
||||||
title: string,
|
title: string,
|
||||||
items: TempUpload[],
|
items: T[],
|
||||||
emptyText: string,
|
emptyText: string,
|
||||||
|
render: (item: T) => ReactNode,
|
||||||
action?: ReactNode,
|
action?: ReactNode,
|
||||||
) => (
|
) => (
|
||||||
<section className="flex min-w-0 flex-col gap-3">
|
<section className="flex min-w-0 flex-col gap-3">
|
||||||
@@ -1148,17 +1262,7 @@ export default function UploadPage() {
|
|||||||
{emptyText}
|
{emptyText}
|
||||||
</p>
|
</p>
|
||||||
) : (
|
) : (
|
||||||
<div className="flex flex-col gap-3">
|
<div className="flex flex-col gap-3">{items.map(render)}</div>
|
||||||
{items.map((temp) => (
|
|
||||||
<TempCard
|
|
||||||
key={temp.temp_id}
|
|
||||||
temp={temp}
|
|
||||||
onOpen={() => setModalId(temp.temp_id)}
|
|
||||||
onRetry={() => retry(temp, "iqdb")}
|
|
||||||
onDismiss={() => dismiss(temp)}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
)}
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
@@ -1387,11 +1491,20 @@ export default function UploadPage() {
|
|||||||
</section>
|
</section>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
<div className="grid gap-6 lg:grid-cols-3">
|
<div className="grid gap-6 lg:grid-cols-4">
|
||||||
{column(
|
{column(
|
||||||
"Pending & Unmatched",
|
"Pending & Unmatched",
|
||||||
pending,
|
pending,
|
||||||
"Nothing waiting.",
|
"Nothing waiting.",
|
||||||
|
(temp) => (
|
||||||
|
<TempCard
|
||||||
|
key={temp.temp_id}
|
||||||
|
temp={temp}
|
||||||
|
onOpen={() => setModalId(temp.temp_id)}
|
||||||
|
onRetry={() => retry(temp, "iqdb")}
|
||||||
|
onDismiss={() => dismiss(temp)}
|
||||||
|
/>
|
||||||
|
),
|
||||||
pending.length > 0 ? (
|
pending.length > 0 ? (
|
||||||
<span className="flex items-center gap-1.5">
|
<span className="flex items-center gap-1.5">
|
||||||
<Button
|
<Button
|
||||||
@@ -1407,7 +1520,7 @@ export default function UploadPage() {
|
|||||||
variant="danger"
|
variant="danger"
|
||||||
className="px-2 py-0.5 text-[11px]"
|
className="px-2 py-0.5 text-[11px]"
|
||||||
disabled={discarding}
|
disabled={discarding}
|
||||||
onClick={() => void discardAll(pending, "discard")}
|
onClick={() => void discardAll(pending)}
|
||||||
title="Delete every staged file in this column"
|
title="Delete every staged file in this column"
|
||||||
>
|
>
|
||||||
<Trash2 className="h-3.5 w-3.5" />
|
<Trash2 className="h-3.5 w-3.5" />
|
||||||
@@ -1420,12 +1533,21 @@ export default function UploadPage() {
|
|||||||
"Visual Similarity Detected",
|
"Visual Similarity Detected",
|
||||||
visual,
|
visual,
|
||||||
"No IQDB matches right now.",
|
"No IQDB matches right now.",
|
||||||
|
(temp) => (
|
||||||
|
<TempCard
|
||||||
|
key={temp.temp_id}
|
||||||
|
temp={temp}
|
||||||
|
onOpen={() => setModalId(temp.temp_id)}
|
||||||
|
onRetry={() => retry(temp, "iqdb")}
|
||||||
|
onDismiss={() => dismiss(temp)}
|
||||||
|
/>
|
||||||
|
),
|
||||||
visual.length > 0 ? (
|
visual.length > 0 ? (
|
||||||
<Button
|
<Button
|
||||||
variant="danger"
|
variant="danger"
|
||||||
className="px-2 py-0.5 text-[11px]"
|
className="px-2 py-0.5 text-[11px]"
|
||||||
disabled={discarding}
|
disabled={discarding}
|
||||||
onClick={() => void discardAll(visual, "discard")}
|
onClick={() => void discardAll(visual)}
|
||||||
title="Delete every staged file in this column"
|
title="Delete every staged file in this column"
|
||||||
>
|
>
|
||||||
<Trash2 className="h-3.5 w-3.5" />
|
<Trash2 className="h-3.5 w-3.5" />
|
||||||
@@ -1435,20 +1557,48 @@ export default function UploadPage() {
|
|||||||
)}
|
)}
|
||||||
{column(
|
{column(
|
||||||
"Auto-uploaded & Indexed",
|
"Auto-uploaded & Indexed",
|
||||||
completed,
|
completions,
|
||||||
"Nothing indexed yet.",
|
"Nothing indexed while you were watching.",
|
||||||
completed.length > 0 ? (
|
(completion) => (
|
||||||
|
<CompletionCard key={completion.id} completion={completion} />
|
||||||
|
),
|
||||||
|
completions.length > 0 ? (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
disabled={discarding}
|
onClick={() => setCompletions([])}
|
||||||
onClick={() => void discardAll(completed, "dismiss")}
|
title="Clear these notifications (nothing is stored)"
|
||||||
title="Dismiss every indexed record"
|
|
||||||
className="rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
className="rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
||||||
>
|
>
|
||||||
dismiss all
|
clear
|
||||||
</button>
|
</button>
|
||||||
) : null,
|
) : null,
|
||||||
)}
|
)}
|
||||||
|
{column(
|
||||||
|
"Failed",
|
||||||
|
failed,
|
||||||
|
"No failed uploads.",
|
||||||
|
(temp) => (
|
||||||
|
<TempCard
|
||||||
|
key={temp.temp_id}
|
||||||
|
temp={temp}
|
||||||
|
onOpen={() => setModalId(temp.temp_id)}
|
||||||
|
onRetry={() => retry(temp)}
|
||||||
|
onDismiss={() => dismiss(temp)}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
failed.length > 0 ? (
|
||||||
|
<Button
|
||||||
|
variant="danger"
|
||||||
|
className="px-2 py-0.5 text-[11px]"
|
||||||
|
disabled={discarding}
|
||||||
|
onClick={() => void discardAll(failed)}
|
||||||
|
title="Delete every failed staged file in this column"
|
||||||
|
>
|
||||||
|
<Trash2 className="h-3.5 w-3.5" />
|
||||||
|
discard all
|
||||||
|
</Button>
|
||||||
|
) : null,
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{modalId ? (
|
{modalId ? (
|
||||||
|
|||||||
@@ -388,6 +388,28 @@ export interface TempUpload {
|
|||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** One upload the pipeline indexed while the page was watching. */
|
||||||
|
export interface UploadCompletion {
|
||||||
|
id: string;
|
||||||
|
filename: string;
|
||||||
|
j_id: string;
|
||||||
|
resolution: string;
|
||||||
|
post_id: number | null;
|
||||||
|
thumbnail_url: string | null;
|
||||||
|
at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Response of POST /api/uploads/ — a pending row or an instant completion. */
|
||||||
|
export interface UploadStagingResult {
|
||||||
|
temp_id: string;
|
||||||
|
original_filename: string;
|
||||||
|
status: "pending" | "visual_match" | "completed" | "error";
|
||||||
|
resolution: string;
|
||||||
|
library_j_id: string | null;
|
||||||
|
preview_url: string | null;
|
||||||
|
e621_post_id?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
/** Progress of the server-side staged-upload pipeline. */
|
/** Progress of the server-side staged-upload pipeline. */
|
||||||
export interface UploadStatus {
|
export interface UploadStatus {
|
||||||
status: "idle" | "running" | "paused" | "error";
|
status: "idle" | "running" | "paused" | "error";
|
||||||
@@ -400,6 +422,8 @@ export interface UploadStatus {
|
|||||||
error: string;
|
error: string;
|
||||||
outstanding: number;
|
outstanding: number;
|
||||||
waiting: { md5: number; visual: number; iqdb: number };
|
waiting: { md5: number; visual: number; iqdb: number };
|
||||||
|
/** Session notification feed: indexed uploads, newest first. */
|
||||||
|
recent_completions: UploadCompletion[];
|
||||||
updated_at: string | null;
|
updated_at: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user