Async Download to Library with progress; Download to client

Backend:
- DownloadTask model + background thread runner: streams the file with
  progress (%, bytes, speed) and a cancel flag, then indexes it, names it
  J-<id>.<ext> and applies the e621 metadata
- DownloadTaskViewSet (create/retrieve/cancel) replaces the synchronous
  endpoint; the status footer's worker counts now reflect download jobs
- Client download proxy (/api/online/file/) streams an e621 original to
  the browser with Content-Disposition: attachment, restricted to the
  configured e621 CDN hosts so it cannot be used as an open proxy

Frontend:
- Online detail: progress bar with percentage, transferred size, speed
  and cancel while downloading; success links to the new J-ID
- New 'Download to client' button available to everyone (guests too)
This commit is contained in:
2026-09-17 12:06:15 -05:00
parent bf00cf36a2
commit 6962e483fc
11 changed files with 509 additions and 63 deletions
+122
View File
@@ -0,0 +1,122 @@
"""Background 'Download to Library' jobs with progress tracking."""
import logging
import threading
import time
from pathlib import Path
from urllib.parse import urlparse
from django.conf import settings
from django.utils import timezone
from . import services
from .models import DownloadTask
logger = logging.getLogger(__name__)
def start_download_task(task_id):
thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True)
thread.start()
def run_download_task(task_id):
task = DownloadTask.objects.filter(id=task_id).first()
if task is None:
return
folder = Path(settings.WATCHED_FOLDER)
name = (
task.filename
or Path(urlparse(task.url).path).name
or f"post-{task.post_id or 'download'}"
)
destination = services.unique_destination(folder, name)
state = {
"progress_at": 0.0,
"bytes_at": 0,
"cancel_at": 0.0,
"cancelled": False,
}
def should_cancel():
now = time.monotonic()
if now - state["cancel_at"] >= 1.0:
state["cancelled"] = DownloadTask.objects.filter(
id=task_id, cancelled=True
).exists()
state["cancel_at"] = now
return state["cancelled"]
def on_progress(downloaded, total):
now = time.monotonic()
if now - state["progress_at"] < 0.5 and (total == 0 or downloaded < total):
return
elapsed = max(now - state["progress_at"], 0.001)
speed = None
if state["bytes_at"] > 0 and downloaded >= state["bytes_at"]:
speed = (downloaded - state["bytes_at"]) / elapsed
state["progress_at"] = now
state["bytes_at"] = downloaded
DownloadTask.objects.filter(id=task_id).update(
downloaded=downloaded,
total=total,
progress=int(downloaded * 100 / total) if total else 0,
speed=speed,
updated_at=timezone.now(),
)
DownloadTask.objects.filter(id=task_id).update(
status=DownloadTask.STATUS_DOWNLOADING, updated_at=timezone.now()
)
try:
services.download_file(
task.url,
destination,
progress_callback=on_progress,
should_cancel=should_cancel,
)
item, _, location, _ = services.index_file(destination, folder)
services.rename_location_to_j_id(item, location)
update_fields = []
if item.uploaded_by_id is None and task.user_id is not None:
item.uploaded_by = task.user
update_fields.append("uploaded_by")
if task.e621_data:
item.e621_post_id = task.post_id
item.e621_data = task.e621_data
update_fields += ["e621_post_id", "e621_data"]
rating = (task.e621_data or {}).get("rating")
if not item.rating and rating in {"s", "q", "e"}:
item.rating = rating
update_fields.append("rating")
if update_fields:
item.save(update_fields=update_fields + ["updated_at"])
DownloadTask.objects.filter(id=task_id).update(
status=DownloadTask.STATUS_COMPLETE,
progress=100,
speed=None,
library_item=item,
updated_at=timezone.now(),
)
except services.DownloadCancelled:
destination.unlink(missing_ok=True)
DownloadTask.objects.filter(id=task_id).update(
status=DownloadTask.STATUS_CANCELLED,
progress=0,
downloaded=0,
speed=None,
updated_at=timezone.now(),
)
except Exception as exc: # noqa: BLE001 - report background failures
destination.unlink(missing_ok=True)
logger.exception("Download task %s failed", task_id)
DownloadTask.objects.filter(id=task_id).update(
status=DownloadTask.STATUS_ERROR,
error=str(exc),
speed=None,
updated_at=timezone.now(),
)