Async Download to Library with progress; Download to client
Backend: - DownloadTask model + background thread runner: streams the file with progress (%, bytes, speed) and a cancel flag, then indexes it, names it J-<id>.<ext> and applies the e621 metadata - DownloadTaskViewSet (create/retrieve/cancel) replaces the synchronous endpoint; the status footer's worker counts now reflect download jobs - Client download proxy (/api/online/file/) streams an e621 original to the browser with Content-Disposition: attachment, restricted to the configured e621 CDN hosts so it cannot be used as an open proxy Frontend: - Online detail: progress bar with percentage, transferred size, speed and cancel while downloading; success links to the new J-ID - New 'Download to client' button available to everyone (guests too)
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
"""Background 'Download to Library' jobs with progress tracking."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from . import services
|
||||
from .models import DownloadTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def start_download_task(task_id):
|
||||
thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def run_download_task(task_id):
|
||||
task = DownloadTask.objects.filter(id=task_id).first()
|
||||
if task is None:
|
||||
return
|
||||
|
||||
folder = Path(settings.WATCHED_FOLDER)
|
||||
name = (
|
||||
task.filename
|
||||
or Path(urlparse(task.url).path).name
|
||||
or f"post-{task.post_id or 'download'}"
|
||||
)
|
||||
destination = services.unique_destination(folder, name)
|
||||
|
||||
state = {
|
||||
"progress_at": 0.0,
|
||||
"bytes_at": 0,
|
||||
"cancel_at": 0.0,
|
||||
"cancelled": False,
|
||||
}
|
||||
|
||||
def should_cancel():
|
||||
now = time.monotonic()
|
||||
if now - state["cancel_at"] >= 1.0:
|
||||
state["cancelled"] = DownloadTask.objects.filter(
|
||||
id=task_id, cancelled=True
|
||||
).exists()
|
||||
state["cancel_at"] = now
|
||||
return state["cancelled"]
|
||||
|
||||
def on_progress(downloaded, total):
|
||||
now = time.monotonic()
|
||||
if now - state["progress_at"] < 0.5 and (total == 0 or downloaded < total):
|
||||
return
|
||||
elapsed = max(now - state["progress_at"], 0.001)
|
||||
speed = None
|
||||
if state["bytes_at"] > 0 and downloaded >= state["bytes_at"]:
|
||||
speed = (downloaded - state["bytes_at"]) / elapsed
|
||||
state["progress_at"] = now
|
||||
state["bytes_at"] = downloaded
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
downloaded=downloaded,
|
||||
total=total,
|
||||
progress=int(downloaded * 100 / total) if total else 0,
|
||||
speed=speed,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_DOWNLOADING, updated_at=timezone.now()
|
||||
)
|
||||
try:
|
||||
services.download_file(
|
||||
task.url,
|
||||
destination,
|
||||
progress_callback=on_progress,
|
||||
should_cancel=should_cancel,
|
||||
)
|
||||
item, _, location, _ = services.index_file(destination, folder)
|
||||
services.rename_location_to_j_id(item, location)
|
||||
|
||||
update_fields = []
|
||||
if item.uploaded_by_id is None and task.user_id is not None:
|
||||
item.uploaded_by = task.user
|
||||
update_fields.append("uploaded_by")
|
||||
if task.e621_data:
|
||||
item.e621_post_id = task.post_id
|
||||
item.e621_data = task.e621_data
|
||||
update_fields += ["e621_post_id", "e621_data"]
|
||||
rating = (task.e621_data or {}).get("rating")
|
||||
if not item.rating and rating in {"s", "q", "e"}:
|
||||
item.rating = rating
|
||||
update_fields.append("rating")
|
||||
if update_fields:
|
||||
item.save(update_fields=update_fields + ["updated_at"])
|
||||
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_COMPLETE,
|
||||
progress=100,
|
||||
speed=None,
|
||||
library_item=item,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
except services.DownloadCancelled:
|
||||
destination.unlink(missing_ok=True)
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_CANCELLED,
|
||||
progress=0,
|
||||
downloaded=0,
|
||||
speed=None,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - report background failures
|
||||
destination.unlink(missing_ok=True)
|
||||
logger.exception("Download task %s failed", task_id)
|
||||
DownloadTask.objects.filter(id=task_id).update(
|
||||
status=DownloadTask.STATUS_ERROR,
|
||||
error=str(exc),
|
||||
speed=None,
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
Reference in New Issue
Block a user