diff --git a/backend/apps/core/views.py b/backend/apps/core/views.py index 0d84860..19d84f7 100644 --- a/backend/apps/core/views.py +++ b/backend/apps/core/views.py @@ -9,11 +9,16 @@ from .system_info import get_os_info def _worker_counts(): - """Active/queued background jobs. + """Active/queued background jobs (download tasks for now).""" + from apps.library.models import DownloadTask - No task pipeline exists yet, so this reports honest zeros for now. - """ - return {"active": 0, "queued": 0} + active = DownloadTask.objects.filter( + status=DownloadTask.STATUS_DOWNLOADING + ).count() + queued = DownloadTask.objects.filter( + status=DownloadTask.STATUS_PENDING + ).count() + return {"active": active, "queued": queued} class StatusView(APIView): diff --git a/backend/apps/library/downloads.py b/backend/apps/library/downloads.py new file mode 100644 index 0000000..e0eac27 --- /dev/null +++ b/backend/apps/library/downloads.py @@ -0,0 +1,122 @@ +"""Background 'Download to Library' jobs with progress tracking.""" + +import logging +import threading +import time +from pathlib import Path +from urllib.parse import urlparse + +from django.conf import settings +from django.utils import timezone + +from . import services +from .models import DownloadTask + +logger = logging.getLogger(__name__) + + +def start_download_task(task_id): + thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True) + thread.start() + + +def run_download_task(task_id): + task = DownloadTask.objects.filter(id=task_id).first() + if task is None: + return + + folder = Path(settings.WATCHED_FOLDER) + name = ( + task.filename + or Path(urlparse(task.url).path).name + or f"post-{task.post_id or 'download'}" + ) + destination = services.unique_destination(folder, name) + + state = { + "progress_at": 0.0, + "bytes_at": 0, + "cancel_at": 0.0, + "cancelled": False, + } + + def should_cancel(): + now = time.monotonic() + if now - state["cancel_at"] >= 1.0: + state["cancelled"] = DownloadTask.objects.filter( + id=task_id, cancelled=True + ).exists() + state["cancel_at"] = now + return state["cancelled"] + + def on_progress(downloaded, total): + now = time.monotonic() + if now - state["progress_at"] < 0.5 and (total == 0 or downloaded < total): + return + elapsed = max(now - state["progress_at"], 0.001) + speed = None + if state["bytes_at"] > 0 and downloaded >= state["bytes_at"]: + speed = (downloaded - state["bytes_at"]) / elapsed + state["progress_at"] = now + state["bytes_at"] = downloaded + DownloadTask.objects.filter(id=task_id).update( + downloaded=downloaded, + total=total, + progress=int(downloaded * 100 / total) if total else 0, + speed=speed, + updated_at=timezone.now(), + ) + + DownloadTask.objects.filter(id=task_id).update( + status=DownloadTask.STATUS_DOWNLOADING, updated_at=timezone.now() + ) + try: + services.download_file( + task.url, + destination, + progress_callback=on_progress, + should_cancel=should_cancel, + ) + item, _, location, _ = services.index_file(destination, folder) + services.rename_location_to_j_id(item, location) + + update_fields = [] + if item.uploaded_by_id is None and task.user_id is not None: + item.uploaded_by = task.user + update_fields.append("uploaded_by") + if task.e621_data: + item.e621_post_id = task.post_id + item.e621_data = task.e621_data + update_fields += ["e621_post_id", "e621_data"] + rating = (task.e621_data or {}).get("rating") + if not item.rating and rating in {"s", "q", "e"}: + item.rating = rating + update_fields.append("rating") + if update_fields: + item.save(update_fields=update_fields + ["updated_at"]) + + DownloadTask.objects.filter(id=task_id).update( + status=DownloadTask.STATUS_COMPLETE, + progress=100, + speed=None, + library_item=item, + updated_at=timezone.now(), + ) + except services.DownloadCancelled: + destination.unlink(missing_ok=True) + DownloadTask.objects.filter(id=task_id).update( + status=DownloadTask.STATUS_CANCELLED, + progress=0, + downloaded=0, + speed=None, + updated_at=timezone.now(), + ) + except Exception as exc: # noqa: BLE001 - report background failures + destination.unlink(missing_ok=True) + logger.exception("Download task %s failed", task_id) + DownloadTask.objects.filter(id=task_id).update( + status=DownloadTask.STATUS_ERROR, + error=str(exc), + speed=None, + updated_at=timezone.now(), + ) diff --git a/backend/apps/library/migrations/0005_downloadtask.py b/backend/apps/library/migrations/0005_downloadtask.py new file mode 100644 index 0000000..ae8dfe2 --- /dev/null +++ b/backend/apps/library/migrations/0005_downloadtask.py @@ -0,0 +1,41 @@ +# Generated by Django 6.1.1 on 2026-09-17 17:03 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('library', '0004_tempupload'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='DownloadTask', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('post_id', models.IntegerField(blank=True, null=True)), + ('url', models.CharField(max_length=1000)), + ('filename', models.CharField(blank=True, default='', max_length=255)), + ('e621_data', models.JSONField(blank=True, null=True)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('downloading', 'Downloading'), ('complete', 'Complete'), ('error', 'Error'), ('cancelled', 'Cancelled')], default='pending', max_length=20)), + ('progress', models.IntegerField(default=0)), + ('downloaded', models.BigIntegerField(default=0)), + ('total', models.BigIntegerField(default=0)), + ('speed', models.FloatField(blank=True, null=True)), + ('error', models.TextField(blank=True, default='')), + ('cancelled', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('library_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to='library.mediaitem')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['-created_at'], + }, + ), + ] diff --git a/backend/apps/library/models.py b/backend/apps/library/models.py index 9705e4e..cd80e4c 100644 --- a/backend/apps/library/models.py +++ b/backend/apps/library/models.py @@ -115,3 +115,57 @@ class TempUpload(models.Model): def __str__(self): return f"{self.original_filename} ({self.status})" + + +class DownloadTask(models.Model): + """A background 'Download to Library' job with progress tracking.""" + + STATUS_PENDING = "pending" + STATUS_DOWNLOADING = "downloading" + STATUS_COMPLETE = "complete" + STATUS_ERROR = "error" + STATUS_CANCELLED = "cancelled" + STATUS_CHOICES = [ + (STATUS_PENDING, "Pending"), + (STATUS_DOWNLOADING, "Downloading"), + (STATUS_COMPLETE, "Complete"), + (STATUS_ERROR, "Error"), + (STATUS_CANCELLED, "Cancelled"), + ] + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + user = models.ForeignKey( + settings.AUTH_USER_MODEL, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="download_tasks", + ) + post_id = models.IntegerField(null=True, blank=True) + url = models.CharField(max_length=1000) + filename = models.CharField(max_length=255, blank=True, default="") + e621_data = models.JSONField(null=True, blank=True) + status = models.CharField( + max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING + ) + progress = models.IntegerField(default=0) + downloaded = models.BigIntegerField(default=0) + total = models.BigIntegerField(default=0) + speed = models.FloatField(null=True, blank=True) + error = models.TextField(blank=True, default="") + cancelled = models.BooleanField(default=False) + library_item = models.ForeignKey( + MediaItem, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="download_tasks", + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + class Meta: + ordering = ["-created_at"] + + def __str__(self): + return f"Download {self.id} ({self.status})" diff --git a/backend/apps/library/serializers.py b/backend/apps/library/serializers.py index d80afd2..71784e1 100644 --- a/backend/apps/library/serializers.py +++ b/backend/apps/library/serializers.py @@ -4,7 +4,7 @@ from pathlib import Path from django.core import signing from rest_framework import serializers -from .models import MediaItem, MediaLocation, TempUpload +from .models import MediaItem, MediaLocation, TempUpload, DownloadTask from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS @@ -169,3 +169,29 @@ class TempUploadSerializer(serializers.ModelSerializer): salt=MEDIA_FILE_SALT, ) return f"/api/files/J-{item.id}/{action}/?sig={signature}" + + +class DownloadTaskSerializer(serializers.ModelSerializer): + task_id = serializers.UUIDField(source="id", read_only=True) + library_j_id = serializers.SerializerMethodField() + + class Meta: + model = DownloadTask + fields = [ + "task_id", + "post_id", + "filename", + "status", + "progress", + "downloaded", + "total", + "speed", + "error", + "library_j_id", + "created_at", + "updated_at", + ] + read_only_fields = fields + + def get_library_j_id(self, obj): + return f"J-{obj.library_item_id}" if obj.library_item_id else None diff --git a/backend/apps/library/services.py b/backend/apps/library/services.py index f42d3cf..1622244 100644 --- a/backend/apps/library/services.py +++ b/backend/apps/library/services.py @@ -189,17 +189,32 @@ def serve_file(request, path, download=False): return response -def download_file(url, destination): +class DownloadCancelled(Exception): + """Raised when a streamed download is cancelled by the user.""" + + +def download_file( + url, destination, progress_callback=None, should_cancel=None +): """Stream a remote file into destination (used by Download to Library).""" import requests headers = {"User-Agent": settings.USER_AGENT} with requests.get(url, headers=headers, stream=True, timeout=120) as response: response.raise_for_status() + total = int(response.headers.get("content-length") or 0) + downloaded = 0 with open(destination, "wb") as handle: for chunk in response.iter_content(chunk_size=CHUNK_SIZE): + if should_cancel is not None and should_cancel(): + raise DownloadCancelled("download cancelled") if chunk: handle.write(chunk) + downloaded += len(chunk) + if progress_callback is not None: + progress_callback(downloaded, total) + if progress_callback is not None: + progress_callback(downloaded, total or downloaded) E621_DESCRIPTION_LIMIT = 20000 diff --git a/backend/apps/library/urls.py b/backend/apps/library/urls.py index 845401e..76b7268 100644 --- a/backend/apps/library/urls.py +++ b/backend/apps/library/urls.py @@ -2,17 +2,14 @@ from django.urls import include, path from rest_framework.routers import DefaultRouter from .uploads import TempUploadViewSet -from .views import DownloadToLibraryView, MediaItemViewSet +from .views import ClientDownloadView, DownloadTaskViewSet, MediaItemViewSet router = DefaultRouter() router.register("files", MediaItemViewSet, basename="file") router.register("uploads", TempUploadViewSet, basename="upload") +router.register("online/downloads", DownloadTaskViewSet, basename="download") urlpatterns = [ path("", include(router.urls)), - path( - "online/downloads/", - DownloadToLibraryView.as_view(), - name="download_to_library", - ), + path("online/file/", ClientDownloadView.as_view(), name="client_download"), ] diff --git a/backend/apps/library/views.py b/backend/apps/library/views.py index 475edd0..c60fde5 100644 --- a/backend/apps/library/views.py +++ b/backend/apps/library/views.py @@ -5,8 +5,10 @@ from urllib.parse import urlparse from django.conf import settings from django.core import signing from django.db.models import Min -from django.http import Http404 +from django.http import Http404, StreamingHttpResponse from django.shortcuts import get_object_or_404 +from django.utils import timezone +from django.utils.text import get_valid_filename from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly @@ -14,9 +16,10 @@ from rest_framework.response import Response from rest_framework.views import APIView from . import services -from .models import MediaItem +from .downloads import start_download_task +from .models import DownloadTask, MediaItem from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly -from .serializers import MediaItemSerializer +from .serializers import DownloadTaskSerializer, MediaItemSerializer LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"} MD5_RE = re.compile(r"[0-9a-fA-F]{32}") @@ -188,59 +191,121 @@ class MediaItemViewSet( ) -class DownloadToLibraryView(APIView): - """Fetch an e621 file into the watched folder and index it.""" +class DownloadTaskViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Background 'Download to Library' jobs.""" + serializer_class = DownloadTaskSerializer permission_classes = [CanUpload] + http_method_names = ["get", "post", "head", "options"] - def post(self, request): + def get_queryset(self): + queryset = DownloadTask.objects.select_related("library_item") + user = self.request.user + if not (user.is_staff or user.is_superuser): + queryset = queryset.filter(user=user) + return queryset + + def create(self, request): url = str(request.data.get("url") or "").strip() post_id = request.data.get("post_id") filename = str(request.data.get("filename") or "").strip() - post_payload = request.data.get("post") if not url.startswith(("http://", "https://")): return Response( {"detail": "A valid file URL is required."}, status=status.HTTP_400_BAD_REQUEST, ) - folder = Path(settings.WATCHED_FOLDER) - name = ( - filename - or Path(urlparse(url).path).name - or f"post-{post_id or 'download'}" + trimmed = services.trim_e621_post(request.data.get("post")) + resolved_post_id = None + if trimmed is not None and trimmed.get("id") is not None: + resolved_post_id = int(trimmed["id"]) + elif post_id is not None and str(post_id).isdigit(): + resolved_post_id = int(post_id) + + task = DownloadTask.objects.create( + user=request.user, + post_id=resolved_post_id, + url=url, + filename=filename, + e621_data=trimmed, ) - destination = services.unique_destination(folder, name) - try: - services.download_file(url, destination) - item, _, location, _ = services.index_file(destination, folder) - except Exception as exc: # noqa: BLE001 - report download failures - destination.unlink(missing_ok=True) + start_download_task(task.id) + return Response( + self.get_serializer(task).data, status=status.HTTP_202_ACCEPTED + ) + + @action(detail=True, methods=["post"]) + def cancel(self, request, pk=None): + task = self.get_object() + if task.status not in { + DownloadTask.STATUS_PENDING, + DownloadTask.STATUS_DOWNLOADING, + }: return Response( - {"detail": f"Download failed: {exc}"}, + {"detail": "Task is not running."}, + status=status.HTTP_400_BAD_REQUEST, + ) + DownloadTask.objects.filter(id=task.id).update( + cancelled=True, updated_at=timezone.now() + ) + return Response({"success": True}) + + +class ClientDownloadView(APIView): + """Stream an e621 file straight to the browser (no library write).""" + + permission_classes = [AllowAny] + + def get(self, request): + url = str(request.query_params.get("url") or "").strip() + filename = str(request.query_params.get("filename") or "").strip() + parsed = urlparse(url) + if ( + parsed.scheme not in {"http", "https"} + or parsed.hostname not in settings.E621_MEDIA_HOSTS + ): + return Response( + {"detail": "URL not allowed."}, status=status.HTTP_400_BAD_REQUEST, ) - update_fields = [] - if item.uploaded_by_id is None: - item.uploaded_by = request.user - update_fields.append("uploaded_by") + import requests - trimmed = services.trim_e621_post(post_payload) - if trimmed is not None: - if trimmed.get("id") is not None: - item.e621_post_id = int(trimmed["id"]) - elif post_id and str(post_id).isdigit(): - item.e621_post_id = int(post_id) - item.e621_data = trimmed - update_fields += ["e621_post_id", "e621_data"] - if not item.rating and trimmed.get("rating") in {"s", "q", "e"}: - item.rating = trimmed["rating"] - update_fields.append("rating") + try: + upstream = requests.get( + url, + headers={"User-Agent": settings.USER_AGENT}, + stream=True, + timeout=60, + ) + upstream.raise_for_status() + except requests.RequestException as exc: + return Response( + {"detail": f"Could not fetch the file: {exc}"}, + status=status.HTTP_502_BAD_GATEWAY, + ) - if update_fields: - item.save(update_fields=update_fields + ["updated_at"]) - services.rename_location_to_j_id(item, location) - return Response( - MediaItemSerializer(item, context={"request": request}).data, - status=status.HTTP_201_CREATED, + content_type = upstream.headers.get( + "Content-Type", "application/octet-stream" ) + name = get_valid_filename( + filename or Path(parsed.path).name or "download" + ) + + def stream(): + try: + for chunk in upstream.iter_content(chunk_size=64 * 1024): + if chunk: + yield chunk + finally: + upstream.close() + + response = StreamingHttpResponse(stream(), content_type=content_type) + response["Content-Disposition"] = f'attachment; filename="{name}"' + length = upstream.headers.get("Content-Length") + if length: + response["Content-Length"] = length + return response diff --git a/backend/config/settings.py b/backend/config/settings.py index 70e9595..c84b06a 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -144,6 +144,15 @@ WATCHED_FOLDER = str(WATCHED_FOLDER) E621_BASE_URL = os.getenv("E621_BASE_URL", "https://e621.net").rstrip("/") USER_AGENT = os.getenv("USER_AGENT", "J621/0.1 (by J621 on e621)") +# Hosts the client-download proxy is allowed to stream from. +E621_MEDIA_HOSTS = [ + host.strip() + for host in os.getenv( + "E621_MEDIA_HOSTS", + "static1.e621.net,static2.e621.net,static3.e621.net", + ).split(",") + if host.strip() +] # Guest visibility: e621's anonymous default blacklist is mirrored into the # cache by `manage.py refresh_guest_blacklist`. This fallback is used until diff --git a/frontend/src/features/online/OnlineDetail.tsx b/frontend/src/features/online/OnlineDetail.tsx index 74dc7bb..1014bb9 100644 --- a/frontend/src/features/online/OnlineDetail.tsx +++ b/frontend/src/features/online/OnlineDetail.tsx @@ -7,13 +7,13 @@ import { Heart, HeartOff, } from "lucide-react"; -import { useMemo } from "react"; +import { useMemo, useEffect, useState } from "react"; import { Link, useLocation, useNavigate } from "react-router-dom"; import { Button, EmptyState, Spinner, linkButtonClass } from "@/components/ui"; import { TagChip } from "@/components/TagChip"; import { DetailShell } from "@/features/detail/DetailShell"; -import { api } from "@/lib/api"; +import { api, errorMessage } from "@/lib/api"; import { cn } from "@/lib/cn"; import { renderDText } from "@/lib/dtext"; import { @@ -27,7 +27,7 @@ import { TAG_CATEGORY_ORDER, } from "@/lib/e621"; import { formatBytes, formatDate } from "@/lib/format"; -import type { MediaItem } from "@/lib/types"; +import type { DownloadTask } from "@/lib/types"; import { useAuth } from "@/store/auth"; import { useE621 } from "@/store/e621"; @@ -116,9 +116,11 @@ export function OnlineDetail({ postId }: { postId: number }) { }, }); + const [downloadTaskId, setDownloadTaskId] = useState(null); + const downloadMutation = useMutation({ mutationFn: (target: NonNullable) => - api("/api/online/downloads/", { + api("/api/online/downloads/", { method: "POST", json: { post_id: target.id, @@ -127,12 +129,36 @@ export function OnlineDetail({ postId }: { postId: number }) { post: target, }, }), - onSuccess: (item) => { - void queryClient.invalidateQueries({ queryKey: ["library-lookup"] }); - void queryClient.invalidateQueries({ queryKey: ["files"] }); - navigate(`/detail/${item.j_id}`); + onSuccess: (task) => setDownloadTaskId(task.task_id), + }); + + const taskQuery = useQuery({ + queryKey: ["download-task", downloadTaskId], + queryFn: () => + api(`/api/online/downloads/${downloadTaskId}/`), + enabled: Boolean(downloadTaskId), + refetchInterval: (query) => { + const status = query.state.data?.status; + return status === "pending" || status === "downloading" ? 700 : false; }, }); + const downloadTask = taskQuery.data; + const downloadActive = + downloadTask?.status === "pending" || downloadTask?.status === "downloading"; + + useEffect(() => { + if (downloadTask?.status === "complete") { + void queryClient.invalidateQueries({ queryKey: ["library-lookup"] }); + void queryClient.invalidateQueries({ queryKey: ["files"] }); + } + }, [downloadTask?.status, queryClient]); + + function cancelDownload() { + if (!downloadTaskId) return; + void api(`/api/online/downloads/${downloadTaskId}/cancel/`, { + method: "POST", + }); + } if (query.isPending) { return ( @@ -163,6 +189,11 @@ export function OnlineDetail({ postId }: { postId: number }) { const imageSrc = post.sample.has && post.sample.url ? post.sample.url : post.file.url; const canUpload = Boolean(user?.can_upload); + const clientDownloadUrl = post.file.url + ? `/api/online/file/?url=${encodeURIComponent(post.file.url)}&filename=${encodeURIComponent( + `${post.file.md5}.${post.file.ext}`, + )}` + : null; return ( downloadMutation.mutate(post)} title={ @@ -223,12 +257,22 @@ export function OnlineDetail({ postId }: { postId: number }) { } > - {downloadMutation.isPending + {downloadMutation.isPending || downloadActive ? "Downloading…" : "Download to Library"} )} + {clientDownloadUrl ? ( + + Download to client + + ) : null} + + {downloadTask && + (downloadTask.status === "pending" || + downloadTask.status === "downloading") ? ( +
+
+ + Downloading to the library… {downloadTask.progress}% + + + {formatBytes(downloadTask.downloaded)} + {downloadTask.total > 0 + ? ` / ${formatBytes(downloadTask.total)}` + : ""} + {downloadTask.speed + ? ` · ${(downloadTask.speed / 1024 / 1024).toFixed(1)} MB/s` + : ""} + +
+
+
+
+ +
+ ) : downloadTask?.status === "complete" ? ( +

+ + Downloaded to the library + {downloadTask.library_j_id ? ( + + {downloadTask.library_j_id} + + ) : null} +

+ ) : downloadTask?.status === "cancelled" ? ( +

Download cancelled.

+ ) : downloadTask?.status === "error" ? ( +

+ {downloadTask.error || "Download failed."} +

+ ) : null} + {favoriteMutation.isError ? (

{e621ErrorMessage(favoriteMutation.error)} @@ -246,7 +343,7 @@ export function OnlineDetail({ postId }: { postId: number }) { ) : null} {downloadMutation.isError ? (

- {e621ErrorMessage(downloadMutation.error)} + {errorMessage(downloadMutation.error)}

) : null} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index e4bec13..417c686 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -107,6 +107,21 @@ export interface FileNeighbors { next: FileNeighbor | null; } +export interface DownloadTask { + task_id: string; + post_id: number | null; + filename: string; + status: "pending" | "downloading" | "complete" | "error" | "cancelled"; + progress: number; + downloaded: number; + total: number; + speed: number | null; + error: string; + library_j_id: string | null; + created_at: string; + updated_at: string; +} + export interface E621IqdbCandidate { post_id: number | null; score: number | null;