Async Download to Library with progress; Download to client
Backend: - DownloadTask model + background thread runner: streams the file with progress (%, bytes, speed) and a cancel flag, then indexes it, names it J-<id>.<ext> and applies the e621 metadata - DownloadTaskViewSet (create/retrieve/cancel) replaces the synchronous endpoint; the status footer's worker counts now reflect download jobs - Client download proxy (/api/online/file/) streams an e621 original to the browser with Content-Disposition: attachment, restricted to the configured e621 CDN hosts so it cannot be used as an open proxy Frontend: - Online detail: progress bar with percentage, transferred size, speed and cancel while downloading; success links to the new J-ID - New 'Download to client' button available to everyone (guests too)
This commit is contained in:
@@ -9,11 +9,16 @@ from .system_info import get_os_info
|
||||
|
||||
|
||||
def _worker_counts():
|
||||
"""Active/queued background jobs.
|
||||
"""Active/queued background jobs (download tasks for now)."""
|
||||
from apps.library.models import DownloadTask
|
||||
|
||||
No task pipeline exists yet, so this reports honest zeros for now.
|
||||
"""
|
||||
return {"active": 0, "queued": 0}
|
||||
active = DownloadTask.objects.filter(
|
||||
status=DownloadTask.STATUS_DOWNLOADING
|
||||
).count()
|
||||
queued = DownloadTask.objects.filter(
|
||||
status=DownloadTask.STATUS_PENDING
|
||||
).count()
|
||||
return {"active": active, "queued": queued}
|
||||
|
||||
|
||||
class StatusView(APIView):
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Background 'Download to Library' jobs with progress tracking."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from . import services
|
||||
from .models import DownloadTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def start_download_task(task_id):
|
||||
thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def run_download_task(task_id):
|
||||
task = DownloadTask.objects.filter(id=task_id).first()
|
||||
if task is None:
|
||||
return
|
||||
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
name = (
|
||||
task.filename
|
||||
or Path(urlparse(task.url).path).name
|
||||
or f"post-{task.post_id or 'download'}"
|
||||
)
|
||||
destination = services.unique_destination(folder, name)
|
||||
|
||||
state = {
|
||||
"progress_at": 0.0,
|
||||
"bytes_at": 0,
|
||||
"cancel_at": 0.0,
|
||||
"cancelled": False,
|
||||
}
|
||||
|
||||
def should_cancel():
|
||||
now = time.monotonic()
|
||||
if now - state["cancel_at"] >= 1.0:
|
||||
state["cancelled"] = DownloadTask.objects.filter(
|
||||
id=task_id, cancelled=True
|
||||
).exists()
|
||||
state["cancel_at"] = now
|
||||
return state["cancelled"]
|
||||
|
||||
def on_progress(downloaded, total):
|
||||
now = time.monotonic()
|
||||
if now - state["progress_at"] < 0.5 and (total == 0 or downloaded < total):
|
||||
return
|
||||
elapsed = max(now - state["progress_at"], 0.001)
|
||||
speed = None
|
||||
if state["bytes_at"] > 0 and downloaded >= state["bytes_at"]:
|
||||
speed = (downloaded - state["bytes_at"]) / elapsed
|
||||
state["progress_at"] = now
|
||||
state["bytes_at"] = downloaded
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
downloaded=downloaded,
|
||||
total=total,
|
||||
progress=int(downloaded * 100 / total) if total else 0,
|
||||
speed=speed,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_DOWNLOADING, updated_at=timezone.now()
|
||||
)
|
||||
try:
|
||||
services.download_file(
|
||||
task.url,
|
||||
destination,
|
||||
progress_callback=on_progress,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
item, _, location, _ = services.index_file(destination, folder)
|
||||
services.rename_location_to_j_id(item, location)
|
||||
|
||||
update_fields = []
|
||||
if item.uploaded_by_id is None and task.user_id is not None:
|
||||
item.uploaded_by = task.user
|
||||
update_fields.append("uploaded_by")
|
||||
if task.e621_data:
|
||||
item.e621_post_id = task.post_id
|
||||
item.e621_data = task.e621_data
|
||||
update_fields += ["e621_post_id", "e621_data"]
|
||||
rating = (task.e621_data or {}).get("rating")
|
||||
if not item.rating and rating in {"s", "q", "e"}:
|
||||
item.rating = rating
|
||||
update_fields.append("rating")
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_COMPLETE,
|
||||
progress=100,
|
||||
speed=None,
|
||||
library_item=item,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
except services.DownloadCancelled:
|
||||
destination.unlink(missing_ok=True)
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_CANCELLED,
|
||||
progress=0,
|
||||
downloaded=0,
|
||||
speed=None,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - report background failures
|
||||
destination.unlink(missing_ok=True)
|
||||
logger.exception("Download task %s failed", task_id)
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_ERROR,
|
||||
error=str(exc),
|
||||
speed=None,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
# Generated by Django 6.1.1 on 2026-09-17 17:03
|
||||
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('library', '0004_tempupload'),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='DownloadTask',
|
||||
fields=[
|
||||
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('post_id', models.IntegerField(blank=True, null=True)),
|
||||
('url', models.CharField(max_length=1000)),
|
||||
('filename', models.CharField(blank=True, default='', max_length=255)),
|
||||
('e621_data', models.JSONField(blank=True, null=True)),
|
||||
('status', models.CharField(choices=[('pending', 'Pending'), ('downloading', 'Downloading'), ('complete', 'Complete'), ('error', 'Error'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
|
||||
('progress', models.IntegerField(default=0)),
|
||||
('downloaded', models.BigIntegerField(default=0)),
|
||||
('total', models.BigIntegerField(default=0)),
|
||||
('speed', models.FloatField(blank=True, null=True)),
|
||||
('error', models.TextField(blank=True, default='')),
|
||||
('cancelled', models.BooleanField(default=False)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('library_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to='library.mediaitem')),
|
||||
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-created_at'],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -115,3 +115,57 @@ class TempUpload(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.original_filename} ({self.status})"
|
||||
|
||||
|
||||
class DownloadTask(models.Model):
|
||||
"""A background 'Download to Library' job with progress tracking."""
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_DOWNLOADING = "downloading"
|
||||
STATUS_COMPLETE = "complete"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CANCELLED = "cancelled"
|
||||
STATUS_CHOICES = [
|
||||
(STATUS_PENDING, "Pending"),
|
||||
(STATUS_DOWNLOADING, "Downloading"),
|
||||
(STATUS_COMPLETE, "Complete"),
|
||||
(STATUS_ERROR, "Error"),
|
||||
(STATUS_CANCELLED, "Cancelled"),
|
||||
]
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="download_tasks",
|
||||
)
|
||||
post_id = models.IntegerField(null=True, blank=True)
|
||||
url = models.CharField(max_length=1000)
|
||||
filename = models.CharField(max_length=255, blank=True, default="")
|
||||
e621_data = models.JSONField(null=True, blank=True)
|
||||
status = models.CharField(
|
||||
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||
)
|
||||
progress = models.IntegerField(default=0)
|
||||
downloaded = models.BigIntegerField(default=0)
|
||||
total = models.BigIntegerField(default=0)
|
||||
speed = models.FloatField(null=True, blank=True)
|
||||
error = models.TextField(blank=True, default="")
|
||||
cancelled = models.BooleanField(default=False)
|
||||
library_item = models.ForeignKey(
|
||||
MediaItem,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="download_tasks",
|
||||
)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
|
||||
def __str__(self):
|
||||
return f"Download {self.id} ({self.status})"
|
||||
|
||||
@@ -4,7 +4,7 @@ from pathlib import Path
|
||||
from django.core import signing
|
||||
from rest_framework import serializers
|
||||
|
||||
from .models import MediaItem, MediaLocation, TempUpload
|
||||
from .models import MediaItem, MediaLocation, TempUpload, DownloadTask
|
||||
from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
|
||||
|
||||
|
||||
@@ -169,3 +169,29 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
||||
salt=MEDIA_FILE_SALT,
|
||||
)
|
||||
return f"/api/files/J-{item.id}/{action}/?sig={signature}"
|
||||
|
||||
|
||||
class DownloadTaskSerializer(serializers.ModelSerializer):
|
||||
task_id = serializers.UUIDField(source="id", read_only=True)
|
||||
library_j_id = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = DownloadTask
|
||||
fields = [
|
||||
"task_id",
|
||||
"post_id",
|
||||
"filename",
|
||||
"status",
|
||||
"progress",
|
||||
"downloaded",
|
||||
"total",
|
||||
"speed",
|
||||
"error",
|
||||
"library_j_id",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
read_only_fields = fields
|
||||
|
||||
def get_library_j_id(self, obj):
|
||||
return f"J-{obj.library_item_id}" if obj.library_item_id else None
|
||||
|
||||
@@ -189,17 +189,32 @@ def serve_file(request, path, download=False):
|
||||
return response
|
||||
|
||||
|
||||
def download_file(url, destination):
|
||||
class DownloadCancelled(Exception):
|
||||
"""Raised when a streamed download is cancelled by the user."""
|
||||
|
||||
|
||||
def download_file(
|
||||
url, destination, progress_callback=None, should_cancel=None
|
||||
):
|
||||
"""Stream a remote file into destination (used by Download to Library)."""
|
||||
import requests
|
||||
|
||||
headers = {"User-Agent": settings.USER_AGENT}
|
||||
with requests.get(url, headers=headers, stream=True, timeout=120) as response:
|
||||
response.raise_for_status()
|
||||
total = int(response.headers.get("content-length") or 0)
|
||||
downloaded = 0
|
||||
with open(destination, "wb") as handle:
|
||||
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
|
||||
if should_cancel is not None and should_cancel():
|
||||
raise DownloadCancelled("download cancelled")
|
||||
if chunk:
|
||||
handle.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if progress_callback is not None:
|
||||
progress_callback(downloaded, total)
|
||||
if progress_callback is not None:
|
||||
progress_callback(downloaded, total or downloaded)
|
||||
|
||||
|
||||
E621_DESCRIPTION_LIMIT = 20000
|
||||
|
||||
@@ -2,17 +2,14 @@ from django.urls import include, path
|
||||
from rest_framework.routers import DefaultRouter
|
||||
|
||||
from .uploads import TempUploadViewSet
|
||||
from .views import DownloadToLibraryView, MediaItemViewSet
|
||||
from .views import ClientDownloadView, DownloadTaskViewSet, MediaItemViewSet
|
||||
|
||||
router = DefaultRouter()
|
||||
router.register("files", MediaItemViewSet, basename="file")
|
||||
router.register("uploads", TempUploadViewSet, basename="upload")
|
||||
router.register("online/downloads", DownloadTaskViewSet, basename="download")
|
||||
|
||||
urlpatterns = [
|
||||
path("", include(router.urls)),
|
||||
path(
|
||||
"online/downloads/",
|
||||
DownloadToLibraryView.as_view(),
|
||||
name="download_to_library",
|
||||
),
|
||||
path("online/file/", ClientDownloadView.as_view(), name="client_download"),
|
||||
]
|
||||
|
||||
+105
-40
@@ -5,8 +5,10 @@ from urllib.parse import urlparse
|
||||
from django.conf import settings
|
||||
from django.core import signing
|
||||
from django.db.models import Min
|
||||
from django.http import Http404
|
||||
from django.http import Http404, StreamingHttpResponse
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.utils import timezone
|
||||
from django.utils.text import get_valid_filename
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
|
||||
@@ -14,9 +16,10 @@ from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from . import services
|
||||
from .models import MediaItem
|
||||
from .downloads import start_download_task
|
||||
from .models import DownloadTask, MediaItem
|
||||
from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly
|
||||
from .serializers import MediaItemSerializer
|
||||
from .serializers import DownloadTaskSerializer, MediaItemSerializer
|
||||
|
||||
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
||||
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
||||
@@ -188,59 +191,121 @@ class MediaItemViewSet(
|
||||
)
|
||||
|
||||
|
||||
class DownloadToLibraryView(APIView):
|
||||
"""Fetch an e621 file into the watched folder and index it."""
|
||||
class DownloadTaskViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.RetrieveModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
"""Background 'Download to Library' jobs."""
|
||||
|
||||
serializer_class = DownloadTaskSerializer
|
||||
permission_classes = [CanUpload]
|
||||
http_method_names = ["get", "post", "head", "options"]
|
||||
|
||||
def post(self, request):
|
||||
def get_queryset(self):
|
||||
queryset = DownloadTask.objects.select_related("library_item")
|
||||
user = self.request.user
|
||||
if not (user.is_staff or user.is_superuser):
|
||||
queryset = queryset.filter(user=user)
|
||||
return queryset
|
||||
|
||||
def create(self, request):
|
||||
url = str(request.data.get("url") or "").strip()
|
||||
post_id = request.data.get("post_id")
|
||||
filename = str(request.data.get("filename") or "").strip()
|
||||
post_payload = request.data.get("post")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return Response(
|
||||
{"detail": "A valid file URL is required."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
name = (
|
||||
filename
|
||||
or Path(urlparse(url).path).name
|
||||
or f"post-{post_id or 'download'}"
|
||||
trimmed = services.trim_e621_post(request.data.get("post"))
|
||||
resolved_post_id = None
|
||||
if trimmed is not None and trimmed.get("id") is not None:
|
||||
resolved_post_id = int(trimmed["id"])
|
||||
elif post_id is not None and str(post_id).isdigit():
|
||||
resolved_post_id = int(post_id)
|
||||
|
||||
task = DownloadTask.objects.create(
|
||||
user=request.user,
|
||||
post_id=resolved_post_id,
|
||||
url=url,
|
||||
filename=filename,
|
||||
e621_data=trimmed,
|
||||
)
|
||||
destination = services.unique_destination(folder, name)
|
||||
try:
|
||||
services.download_file(url, destination)
|
||||
item, _, location, _ = services.index_file(destination, folder)
|
||||
except Exception as exc: # noqa: BLE001 - report download failures
|
||||
destination.unlink(missing_ok=True)
|
||||
start_download_task(task.id)
|
||||
return Response(
|
||||
self.get_serializer(task).data, status=status.HTTP_202_ACCEPTED
|
||||
)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def cancel(self, request, pk=None):
|
||||
task = self.get_object()
|
||||
if task.status not in {
|
||||
DownloadTask.STATUS_PENDING,
|
||||
DownloadTask.STATUS_DOWNLOADING,
|
||||
}:
|
||||
return Response(
|
||||
{"detail": f"Download failed: {exc}"},
|
||||
{"detail": "Task is not running."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
DownloadTask.objects.filter(id=task.id).update(
|
||||
cancelled=True, updated_at=timezone.now()
|
||||
)
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class ClientDownloadView(APIView):
|
||||
"""Stream an e621 file straight to the browser (no library write)."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
|
||||
def get(self, request):
|
||||
url = str(request.query_params.get("url") or "").strip()
|
||||
filename = str(request.query_params.get("filename") or "").strip()
|
||||
parsed = urlparse(url)
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or parsed.hostname not in settings.E621_MEDIA_HOSTS
|
||||
):
|
||||
return Response(
|
||||
{"detail": "URL not allowed."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
|
||||
update_fields = []
|
||||
if item.uploaded_by_id is None:
|
||||
item.uploaded_by = request.user
|
||||
update_fields.append("uploaded_by")
|
||||
import requests
|
||||
|
||||
trimmed = services.trim_e621_post(post_payload)
|
||||
if trimmed is not None:
|
||||
if trimmed.get("id") is not None:
|
||||
item.e621_post_id = int(trimmed["id"])
|
||||
elif post_id and str(post_id).isdigit():
|
||||
item.e621_post_id = int(post_id)
|
||||
item.e621_data = trimmed
|
||||
update_fields += ["e621_post_id", "e621_data"]
|
||||
if not item.rating and trimmed.get("rating") in {"s", "q", "e"}:
|
||||
item.rating = trimmed["rating"]
|
||||
update_fields.append("rating")
|
||||
try:
|
||||
upstream = requests.get(
|
||||
url,
|
||||
headers={"User-Agent": settings.USER_AGENT},
|
||||
stream=True,
|
||||
timeout=60,
|
||||
)
|
||||
upstream.raise_for_status()
|
||||
except requests.RequestException as exc:
|
||||
return Response(
|
||||
{"detail": f"Could not fetch the file: {exc}"},
|
||||
status=status.HTTP_502_BAD_GATEWAY,
|
||||
)
|
||||
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
services.rename_location_to_j_id(item, location)
|
||||
return Response(
|
||||
MediaItemSerializer(item, context={"request": request}).data,
|
||||
status=status.HTTP_201_CREATED,
|
||||
content_type = upstream.headers.get(
|
||||
"Content-Type", "application/octet-stream"
|
||||
)
|
||||
name = get_valid_filename(
|
||||
filename or Path(parsed.path).name or "download"
|
||||
)
|
||||
|
||||
def stream():
|
||||
try:
|
||||
for chunk in upstream.iter_content(chunk_size=64 * 1024):
|
||||
if chunk:
|
||||
yield chunk
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
response = StreamingHttpResponse(stream(), content_type=content_type)
|
||||
response["Content-Disposition"] = f'attachment; filename="{name}"'
|
||||
length = upstream.headers.get("Content-Length")
|
||||
if length:
|
||||
response["Content-Length"] = length
|
||||
return response
|
||||
|
||||
Reference in New Issue
Block a user