Async Download to Library with progress; Download to client
Backend: - DownloadTask model + background thread runner: streams the file with progress (%, bytes, speed) and a cancel flag, then indexes it, names it J-<id>.<ext> and applies the e621 metadata - DownloadTaskViewSet (create/retrieve/cancel) replaces the synchronous endpoint; the status footer's worker counts now reflect download jobs - Client download proxy (/api/online/file/) streams an e621 original to the browser with Content-Disposition: attachment, restricted to the configured e621 CDN hosts so it cannot be used as an open proxy Frontend: - Online detail: progress bar with percentage, transferred size, speed and cancel while downloading; success links to the new J-ID - New 'Download to client' button available to everyone (guests too)
This commit is contained in:
+105
-40
@@ -5,8 +5,10 @@ from urllib.parse import urlparse
|
||||
from django.conf import settings
|
||||
from django.core import signing
|
||||
from django.db.models import Min
|
||||
from django.http import Http404
|
||||
from django.http import Http404, StreamingHttpResponse
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.utils import timezone
|
||||
from django.utils.text import get_valid_filename
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
|
||||
@@ -14,9 +16,10 @@ from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from . import services
|
||||
from .models import MediaItem
|
||||
from .downloads import start_download_task
|
||||
from .models import DownloadTask, MediaItem
|
||||
from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly
|
||||
from .serializers import MediaItemSerializer
|
||||
from .serializers import DownloadTaskSerializer, MediaItemSerializer
|
||||
|
||||
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
||||
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
||||
@@ -188,59 +191,121 @@ class MediaItemViewSet(
|
||||
)
|
||||
|
||||
|
||||
class DownloadToLibraryView(APIView):
|
||||
"""Fetch an e621 file into the watched folder and index it."""
|
||||
class DownloadTaskViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.RetrieveModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
"""Background 'Download to Library' jobs."""
|
||||
|
||||
serializer_class = DownloadTaskSerializer
|
||||
permission_classes = [CanUpload]
|
||||
http_method_names = ["get", "post", "head", "options"]
|
||||
|
||||
def post(self, request):
|
||||
def get_queryset(self):
|
||||
queryset = DownloadTask.objects.select_related("library_item")
|
||||
user = self.request.user
|
||||
if not (user.is_staff or user.is_superuser):
|
||||
queryset = queryset.filter(user=user)
|
||||
return queryset
|
||||
|
||||
def create(self, request):
|
||||
url = str(request.data.get("url") or "").strip()
|
||||
post_id = request.data.get("post_id")
|
||||
filename = str(request.data.get("filename") or "").strip()
|
||||
post_payload = request.data.get("post")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return Response(
|
||||
{"detail": "A valid file URL is required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
name = (
|
||||
filename
|
||||
or Path(urlparse(url).path).name
|
||||
or f"post-{post_id or 'download'}"
|
||||
trimmed = services.trim_e621_post(request.data.get("post"))
|
||||
resolved_post_id = None
|
||||
if trimmed is not None and trimmed.get("id") is not None:
|
||||
resolved_post_id = int(trimmed["id"])
|
||||
elif post_id is not None and str(post_id).isdigit():
|
||||
resolved_post_id = int(post_id)
|
||||
|
||||
task = DownloadTask.objects.create(
|
||||
user=request.user,
|
||||
post_id=resolved_post_id,
|
||||
url=url,
|
||||
filename=filename,
|
||||
e621_data=trimmed,
|
||||
)
|
||||
destination = services.unique_destination(folder, name)
|
||||
try:
|
||||
services.download_file(url, destination)
|
||||
item, _, location, _ = services.index_file(destination, folder)
|
||||
except Exception as exc: # noqa: BLE001 - report download failures
|
||||
destination.unlink(missing_ok=True)
|
||||
start_download_task(task.id)
|
||||
return Response(
|
||||
self.get_serializer(task).data, status=status.HTTP_202_ACCEPTED
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def cancel(self, request, pk=None):
|
||||
task = self.get_object()
|
||||
if task.status not in {
|
||||
DownloadTask.STATUS_PENDING,
|
||||
DownloadTask.STATUS_DOWNLOADING,
|
||||
}:
|
||||
return Response(
|
||||
{"detail": f"Download failed: {exc}"},
|
||||
{"detail": "Task is not running."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
DownloadTask.objects.filter(id=task.id).update(
|
||||
cancelled=True, updated_at=timezone.now()
|
||||
)
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class ClientDownloadView(APIView):
|
||||
"""Stream an e621 file straight to the browser (no library write)."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
|
||||
def get(self, request):
|
||||
url = str(request.query_params.get("url") or "").strip()
|
||||
filename = str(request.query_params.get("filename") or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or parsed.hostname not in settings.E621_MEDIA_HOSTS
|
||||
):
|
||||
return Response(
|
||||
{"detail": "URL not allowed."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
update_fields = []
|
||||
if item.uploaded_by_id is None:
|
||||
item.uploaded_by = request.user
|
||||
update_fields.append("uploaded_by")
|
||||
import requests
|
||||
|
||||
trimmed = services.trim_e621_post(post_payload)
|
||||
if trimmed is not None:
|
||||
if trimmed.get("id") is not None:
|
||||
item.e621_post_id = int(trimmed["id"])
|
||||
elif post_id and str(post_id).isdigit():
|
||||
item.e621_post_id = int(post_id)
|
||||
item.e621_data = trimmed
|
||||
update_fields += ["e621_post_id", "e621_data"]
|
||||
if not item.rating and trimmed.get("rating") in {"s", "q", "e"}:
|
||||
item.rating = trimmed["rating"]
|
||||
update_fields.append("rating")
|
||||
try:
|
||||
upstream = requests.get(
|
||||
url,
|
||||
headers={"User-Agent": settings.USER_AGENT},
|
||||
stream=True,
|
||||
timeout=60,
|
||||
)
|
||||
upstream.raise_for_status()
|
||||
except requests.RequestException as exc:
|
||||
return Response(
|
||||
{"detail": f"Could not fetch the file: {exc}"},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
services.rename_location_to_j_id(item, location)
|
||||
return Response(
|
||||
MediaItemSerializer(item, context={"request": request}).data,
|
||||
status=status.HTTP_201_CREATED,
|
||||
content_type = upstream.headers.get(
|
||||
"Content-Type", "application/octet-stream"
|
||||
)
|
||||
name = get_valid_filename(
|
||||
filename or Path(parsed.path).name or "download"
|
||||
)
|
||||
|
||||
def stream():
|
||||
try:
|
||||
for chunk in upstream.iter_content(chunk_size=64 * 1024):
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
response = StreamingHttpResponse(stream(), content_type=content_type)
|
||||
response["Content-Disposition"] = f'attachment; filename="{name}"'
|
||||
length = upstream.headers.get("Content-Length")
|
||||
if length:
|
||||
response["Content-Length"] = length
|
||||
return response
|
||||
|
||||
Reference in New Issue
Block a user