Files
J621/backend/apps/library/services.py
T
JakeBreath e5cc63b0cc Phase 3: J-IDs, ownership, roles, guest safety, adaptive detail, download
Backend:
- User.role (user/uploader/staff) with can_upload; uploads and downloads
  gated to uploader+; owners and staff can edit their items
- MediaItem.uploaded_by plus J-<id> identity (serializer, admin,
  scan_files --user, first superuser as default owner)
- API resolves J-<id>, bare numeric ids and MD5s; neighbors and lookup
  return j_ids
- Guest safety: mirror e621's anonymous default blacklist into Redis
  (parses comments, negations and wildcards), flag hidden_from_guests
  and filter lists, details and lookups for anonymous users
- POST /api/online/downloads/ writes an e621 file into the watched
  folder and indexes it for the uploader
- MariaDB + Redis via docker compose (host ports 3307/6380), PyMySQL
  driver shim, Redis cache replacing the file cache; SQLite data
  dumped and loaded into MariaDB

Frontend:
- Single /detail/:itemId route with an adaptive shell: J-<id> renders
  the library item, bare numbers render the e621 post
- Legacy /view/<md5> and /online/view/<id> redirect to canonical URLs
- Cards expose J-IDs; library custom-data editor is read-only for
  non-owners
- Role gating: Upload hidden/blocked for regular users, account shows
  the role, guest hint on the library
2026-09-17 10:16:45 -05:00

205 lines
6.2 KiB
Python

import hashlib
import json
import mimetypes
import os
import re
import shutil
import subprocess
from pathlib import Path
from django.conf import settings
from django.http import FileResponse, Http404, HttpResponse
from django.utils.text import get_valid_filename
from .models import MediaItem, MediaLocation
ALLOWED_EXTENSIONS = {".jpg", ".jpeg", ".png", ".gif", ".apng", ".mp4", ".webm"}
VIDEO_EXTENSIONS = {".mp4", ".webm"}
CHUNK_SIZE = 1024 * 1024
RANGE_RE = re.compile(r"bytes=(\d*)-(\d*)$")
def compute_md5(path):
digest = hashlib.md5()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(CHUNK_SIZE), b""):
digest.update(chunk)
return digest.hexdigest()
def index_file(path, folder):
"""Index one file into MediaItem/MediaLocation.
Returns (item, created_item, created_location).
"""
path = Path(path)
folder = Path(folder)
stat = path.stat()
md5 = compute_md5(path)
item, created_item = MediaItem.objects.get_or_create(
md5=md5, defaults={"size": stat.st_size}
)
if not created_item and item.size != stat.st_size:
item.size = stat.st_size
item.save(update_fields=["size", "updated_at"])
location, created_location = MediaLocation.objects.update_or_create(
item=item,
path=str(path),
defaults={
"rel_path": str(path.relative_to(folder)),
"mtime": stat.st_mtime,
},
)
return item, created_item, created_location
def parse_tags(raw):
"""Normalize a comma-separated string or JSON list into a list of tags."""
if raw is None:
return None
if isinstance(raw, (list, tuple)):
values = raw
else:
text = str(raw).strip()
if not text:
return None
if text.startswith("["):
try:
values = json.loads(text)
except json.JSONDecodeError:
values = text.split(",")
else:
values = text.split(",")
cleaned = []
for value in values:
tag = str(value).strip()
if tag and tag not in cleaned:
cleaned.append(tag[:100])
return cleaned
def unique_destination(folder, filename):
"""Return a non-existing path inside folder for the given filename."""
folder = Path(folder)
folder.mkdir(parents=True, exist_ok=True)
filename = get_valid_filename(Path(filename).name) or "upload"
candidate = folder / filename
stem, suffix = candidate.stem, candidate.suffix
counter = 1
while candidate.exists():
candidate = folder / f"{stem}-{counter}{suffix}"
counter += 1
return candidate
class RangeFileWrapper:
"""Iterate over a limited byte range of an open file."""
def __init__(self, file, length, chunk_size=64 * 1024):
self.file = file
self.remaining = length
self.chunk_size = chunk_size
def __iter__(self):
return self
def __next__(self):
if self.remaining <= 0:
raise StopIteration
data = self.file.read(min(self.chunk_size, self.remaining))
if not data:
raise StopIteration
self.remaining -= len(data)
return data
def close(self):
self.file.close()
def serve_file(request, path, download=False):
"""Serve a file with HTTP range support (needed for video seeking)."""
path = Path(path)
if not path.is_file():
raise Http404
size = path.stat().st_size
content_type = mimetypes.guess_type(str(path))[0] or "application/octet-stream"
range_header = request.headers.get("Range", "").strip()
if range_header:
match = RANGE_RE.match(range_header)
if match:
start_raw, end_raw = match.groups()
if start_raw == "" and end_raw:
length = min(int(end_raw), size)
start, end = size - length, size - 1
else:
start = int(start_raw or 0)
end = min(int(end_raw) if end_raw else size - 1, size - 1)
if start >= size or start > end:
response = HttpResponse(status=416)
response["Content-Range"] = f"bytes */{size}"
return response
length = end - start + 1
handle = open(path, "rb")
handle.seek(start)
response = FileResponse(
RangeFileWrapper(handle, length),
status=206,
content_type=content_type,
)
response["Content-Length"] = str(length)
response["Content-Range"] = f"bytes {start}-{end}/{size}"
response["Accept-Ranges"] = "bytes"
return response
response = FileResponse(
open(path, "rb"),
content_type=content_type,
as_attachment=download,
filename=path.name,
)
response["Accept-Ranges"] = "bytes"
return response
def download_file(url, destination):
"""Stream a remote file into destination (used by Download to Library)."""
import requests
headers = {"User-Agent": settings.USER_AGENT}
with requests.get(url, headers=headers, stream=True, timeout=120) as response:
response.raise_for_status()
with open(destination, "wb") as handle:
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
if chunk:
handle.write(chunk)
def generate_video_thumbnail(md5, path):
"""Extract a JPEG thumbnail from a video, cached under MEDIA_ROOT/thumbs."""
if not shutil.which("ffmpeg"):
return None
thumbs_dir = Path(settings.MEDIA_ROOT) / "thumbs"
thumbs_dir.mkdir(parents=True, exist_ok=True)
target = thumbs_dir / f"{md5}.jpg"
if target.exists() and target.stat().st_mtime >= os.path.getmtime(path):
return target
command = [
"ffmpeg",
"-y",
"-ss",
"0.5",
"-i",
str(path),
"-frames:v",
"1",
"-vf",
"scale=480:-2",
"-loglevel",
"error",
str(target),
]
try:
subprocess.run(command, check=True, capture_output=True, timeout=60)
except (subprocess.SubprocessError, OSError):
return None
return target if target.exists() else None