Async Download to Library with progress; Download to client
Backend: - DownloadTask model + background thread runner: streams the file with progress (%, bytes, speed) and a cancel flag, then indexes it, names it J-<id>.<ext> and applies the e621 metadata - DownloadTaskViewSet (create/retrieve/cancel) replaces the synchronous endpoint; the status footer's worker counts now reflect download jobs - Client download proxy (/api/online/file/) streams an e621 original to the browser with Content-Disposition: attachment, restricted to the configured e621 CDN hosts so it cannot be used as an open proxy Frontend: - Online detail: progress bar with percentage, transferred size, speed and cancel while downloading; success links to the new J-ID - New 'Download to client' button available to everyone (guests too)
This commit is contained in:
@@ -9,11 +9,16 @@ from .system_info import get_os_info
|
|||||||
|
|
||||||
|
|
||||||
def _worker_counts():
|
def _worker_counts():
|
||||||
"""Active/queued background jobs.
|
"""Active/queued background jobs (download tasks for now)."""
|
||||||
|
from apps.library.models import DownloadTask
|
||||||
|
|
||||||
No task pipeline exists yet, so this reports honest zeros for now.
|
active = DownloadTask.objects.filter(
|
||||||
"""
|
status=DownloadTask.STATUS_DOWNLOADING
|
||||||
return {"active": 0, "queued": 0}
|
).count()
|
||||||
|
queued = DownloadTask.objects.filter(
|
||||||
|
status=DownloadTask.STATUS_PENDING
|
||||||
|
).count()
|
||||||
|
return {"active": active, "queued": queued}
|
||||||
|
|
||||||
|
|
||||||
class StatusView(APIView):
|
class StatusView(APIView):
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""Background 'Download to Library' jobs with progress tracking."""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from . import services
|
||||||
|
from .models import DownloadTask
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def start_download_task(task_id):
|
||||||
|
thread = threading.Thread(target=run_download_task, args=(task_id,), daemon=True)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
|
||||||
|
def run_download_task(task_id):
|
||||||
|
task = DownloadTask.objects.filter(id=task_id).first()
|
||||||
|
if task is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
folder = Path(settings.WATCHED_FOLDER)
|
||||||
|
name = (
|
||||||
|
task.filename
|
||||||
|
or Path(urlparse(task.url).path).name
|
||||||
|
or f"post-{task.post_id or 'download'}"
|
||||||
|
)
|
||||||
|
destination = services.unique_destination(folder, name)
|
||||||
|
|
||||||
|
state = {
|
||||||
|
"progress_at": 0.0,
|
||||||
|
"bytes_at": 0,
|
||||||
|
"cancel_at": 0.0,
|
||||||
|
"cancelled": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
def should_cancel():
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - state["cancel_at"] >= 1.0:
|
||||||
|
state["cancelled"] = DownloadTask.objects.filter(
|
||||||
|
id=task_id, cancelled=True
|
||||||
|
).exists()
|
||||||
|
state["cancel_at"] = now
|
||||||
|
return state["cancelled"]
|
||||||
|
|
||||||
|
def on_progress(downloaded, total):
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - state["progress_at"] < 0.5 and (total == 0 or downloaded < total):
|
||||||
|
return
|
||||||
|
elapsed = max(now - state["progress_at"], 0.001)
|
||||||
|
speed = None
|
||||||
|
if state["bytes_at"] > 0 and downloaded >= state["bytes_at"]:
|
||||||
|
speed = (downloaded - state["bytes_at"]) / elapsed
|
||||||
|
state["progress_at"] = now
|
||||||
|
state["bytes_at"] = downloaded
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
downloaded=downloaded,
|
||||||
|
total=total,
|
||||||
|
progress=int(downloaded * 100 / total) if total else 0,
|
||||||
|
speed=speed,
|
||||||
|
updated_at=timezone.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
status=DownloadTask.STATUS_DOWNLOADING, updated_at=timezone.now()
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
services.download_file(
|
||||||
|
task.url,
|
||||||
|
destination,
|
||||||
|
progress_callback=on_progress,
|
||||||
|
should_cancel=should_cancel,
|
||||||
|
)
|
||||||
|
item, _, location, _ = services.index_file(destination, folder)
|
||||||
|
services.rename_location_to_j_id(item, location)
|
||||||
|
|
||||||
|
update_fields = []
|
||||||
|
if item.uploaded_by_id is None and task.user_id is not None:
|
||||||
|
item.uploaded_by = task.user
|
||||||
|
update_fields.append("uploaded_by")
|
||||||
|
if task.e621_data:
|
||||||
|
item.e621_post_id = task.post_id
|
||||||
|
item.e621_data = task.e621_data
|
||||||
|
update_fields += ["e621_post_id", "e621_data"]
|
||||||
|
rating = (task.e621_data or {}).get("rating")
|
||||||
|
if not item.rating and rating in {"s", "q", "e"}:
|
||||||
|
item.rating = rating
|
||||||
|
update_fields.append("rating")
|
||||||
|
if update_fields:
|
||||||
|
item.save(update_fields=update_fields + ["updated_at"])
|
||||||
|
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
status=DownloadTask.STATUS_COMPLETE,
|
||||||
|
progress=100,
|
||||||
|
speed=None,
|
||||||
|
library_item=item,
|
||||||
|
updated_at=timezone.now(),
|
||||||
|
)
|
||||||
|
except services.DownloadCancelled:
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
status=DownloadTask.STATUS_CANCELLED,
|
||||||
|
progress=0,
|
||||||
|
downloaded=0,
|
||||||
|
speed=None,
|
||||||
|
updated_at=timezone.now(),
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 - report background failures
|
||||||
|
destination.unlink(missing_ok=True)
|
||||||
|
logger.exception("Download task %s failed", task_id)
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
status=DownloadTask.STATUS_ERROR,
|
||||||
|
error=str(exc),
|
||||||
|
speed=None,
|
||||||
|
updated_at=timezone.now(),
|
||||||
|
)
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
# Generated by Django 6.1.1 on 2026-09-17 17:03
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import uuid
|
||||||
|
from django.conf import settings
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
('library', '0004_tempupload'),
|
||||||
|
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name='DownloadTask',
|
||||||
|
fields=[
|
||||||
|
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||||
|
('post_id', models.IntegerField(blank=True, null=True)),
|
||||||
|
('url', models.CharField(max_length=1000)),
|
||||||
|
('filename', models.CharField(blank=True, default='', max_length=255)),
|
||||||
|
('e621_data', models.JSONField(blank=True, null=True)),
|
||||||
|
('status', models.CharField(choices=[('pending', 'Pending'), ('downloading', 'Downloading'), ('complete', 'Complete'), ('error', 'Error'), ('cancelled', 'Cancelled')], default='pending', max_length=20)),
|
||||||
|
('progress', models.IntegerField(default=0)),
|
||||||
|
('downloaded', models.BigIntegerField(default=0)),
|
||||||
|
('total', models.BigIntegerField(default=0)),
|
||||||
|
('speed', models.FloatField(blank=True, null=True)),
|
||||||
|
('error', models.TextField(blank=True, default='')),
|
||||||
|
('cancelled', models.BooleanField(default=False)),
|
||||||
|
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||||
|
('updated_at', models.DateTimeField(auto_now=True)),
|
||||||
|
('library_item', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to='library.mediaitem')),
|
||||||
|
('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_tasks', to=settings.AUTH_USER_MODEL)),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
'ordering': ['-created_at'],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -115,3 +115,57 @@ class TempUpload(models.Model):
|
|||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
return f"{self.original_filename} ({self.status})"
|
return f"{self.original_filename} ({self.status})"
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadTask(models.Model):
|
||||||
|
"""A background 'Download to Library' job with progress tracking."""
|
||||||
|
|
||||||
|
STATUS_PENDING = "pending"
|
||||||
|
STATUS_DOWNLOADING = "downloading"
|
||||||
|
STATUS_COMPLETE = "complete"
|
||||||
|
STATUS_ERROR = "error"
|
||||||
|
STATUS_CANCELLED = "cancelled"
|
||||||
|
STATUS_CHOICES = [
|
||||||
|
(STATUS_PENDING, "Pending"),
|
||||||
|
(STATUS_DOWNLOADING, "Downloading"),
|
||||||
|
(STATUS_COMPLETE, "Complete"),
|
||||||
|
(STATUS_ERROR, "Error"),
|
||||||
|
(STATUS_CANCELLED, "Cancelled"),
|
||||||
|
]
|
||||||
|
|
||||||
|
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||||
|
user = models.ForeignKey(
|
||||||
|
settings.AUTH_USER_MODEL,
|
||||||
|
null=True,
|
||||||
|
blank=True,
|
||||||
|
on_delete=models.SET_NULL,
|
||||||
|
related_name="download_tasks",
|
||||||
|
)
|
||||||
|
post_id = models.IntegerField(null=True, blank=True)
|
||||||
|
url = models.CharField(max_length=1000)
|
||||||
|
filename = models.CharField(max_length=255, blank=True, default="")
|
||||||
|
e621_data = models.JSONField(null=True, blank=True)
|
||||||
|
status = models.CharField(
|
||||||
|
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||||
|
)
|
||||||
|
progress = models.IntegerField(default=0)
|
||||||
|
downloaded = models.BigIntegerField(default=0)
|
||||||
|
total = models.BigIntegerField(default=0)
|
||||||
|
speed = models.FloatField(null=True, blank=True)
|
||||||
|
error = models.TextField(blank=True, default="")
|
||||||
|
cancelled = models.BooleanField(default=False)
|
||||||
|
library_item = models.ForeignKey(
|
||||||
|
MediaItem,
|
||||||
|
null=True,
|
||||||
|
blank=True,
|
||||||
|
on_delete=models.SET_NULL,
|
||||||
|
related_name="download_tasks",
|
||||||
|
)
|
||||||
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
|
updated_at = models.DateTimeField(auto_now=True)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
ordering = ["-created_at"]
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f"Download {self.id} ({self.status})"
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from pathlib import Path
|
|||||||
from django.core import signing
|
from django.core import signing
|
||||||
from rest_framework import serializers
|
from rest_framework import serializers
|
||||||
|
|
||||||
from .models import MediaItem, MediaLocation, TempUpload
|
from .models import MediaItem, MediaLocation, TempUpload, DownloadTask
|
||||||
from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
|
from .services import MEDIA_FILE_SALT, UPLOAD_FILE_SALT, VIDEO_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
@@ -169,3 +169,29 @@ class TempUploadSerializer(serializers.ModelSerializer):
|
|||||||
salt=MEDIA_FILE_SALT,
|
salt=MEDIA_FILE_SALT,
|
||||||
)
|
)
|
||||||
return f"/api/files/J-{item.id}/{action}/?sig={signature}"
|
return f"/api/files/J-{item.id}/{action}/?sig={signature}"
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadTaskSerializer(serializers.ModelSerializer):
|
||||||
|
task_id = serializers.UUIDField(source="id", read_only=True)
|
||||||
|
library_j_id = serializers.SerializerMethodField()
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
model = DownloadTask
|
||||||
|
fields = [
|
||||||
|
"task_id",
|
||||||
|
"post_id",
|
||||||
|
"filename",
|
||||||
|
"status",
|
||||||
|
"progress",
|
||||||
|
"downloaded",
|
||||||
|
"total",
|
||||||
|
"speed",
|
||||||
|
"error",
|
||||||
|
"library_j_id",
|
||||||
|
"created_at",
|
||||||
|
"updated_at",
|
||||||
|
]
|
||||||
|
read_only_fields = fields
|
||||||
|
|
||||||
|
def get_library_j_id(self, obj):
|
||||||
|
return f"J-{obj.library_item_id}" if obj.library_item_id else None
|
||||||
|
|||||||
@@ -189,17 +189,32 @@ def serve_file(request, path, download=False):
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
def download_file(url, destination):
|
class DownloadCancelled(Exception):
|
||||||
|
"""Raised when a streamed download is cancelled by the user."""
|
||||||
|
|
||||||
|
|
||||||
|
def download_file(
|
||||||
|
url, destination, progress_callback=None, should_cancel=None
|
||||||
|
):
|
||||||
"""Stream a remote file into destination (used by Download to Library)."""
|
"""Stream a remote file into destination (used by Download to Library)."""
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
headers = {"User-Agent": settings.USER_AGENT}
|
headers = {"User-Agent": settings.USER_AGENT}
|
||||||
with requests.get(url, headers=headers, stream=True, timeout=120) as response:
|
with requests.get(url, headers=headers, stream=True, timeout=120) as response:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
total = int(response.headers.get("content-length") or 0)
|
||||||
|
downloaded = 0
|
||||||
with open(destination, "wb") as handle:
|
with open(destination, "wb") as handle:
|
||||||
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
|
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
|
||||||
|
if should_cancel is not None and should_cancel():
|
||||||
|
raise DownloadCancelled("download cancelled")
|
||||||
if chunk:
|
if chunk:
|
||||||
handle.write(chunk)
|
handle.write(chunk)
|
||||||
|
downloaded += len(chunk)
|
||||||
|
if progress_callback is not None:
|
||||||
|
progress_callback(downloaded, total)
|
||||||
|
if progress_callback is not None:
|
||||||
|
progress_callback(downloaded, total or downloaded)
|
||||||
|
|
||||||
|
|
||||||
E621_DESCRIPTION_LIMIT = 20000
|
E621_DESCRIPTION_LIMIT = 20000
|
||||||
|
|||||||
@@ -2,17 +2,14 @@ from django.urls import include, path
|
|||||||
from rest_framework.routers import DefaultRouter
|
from rest_framework.routers import DefaultRouter
|
||||||
|
|
||||||
from .uploads import TempUploadViewSet
|
from .uploads import TempUploadViewSet
|
||||||
from .views import DownloadToLibraryView, MediaItemViewSet
|
from .views import ClientDownloadView, DownloadTaskViewSet, MediaItemViewSet
|
||||||
|
|
||||||
router = DefaultRouter()
|
router = DefaultRouter()
|
||||||
router.register("files", MediaItemViewSet, basename="file")
|
router.register("files", MediaItemViewSet, basename="file")
|
||||||
router.register("uploads", TempUploadViewSet, basename="upload")
|
router.register("uploads", TempUploadViewSet, basename="upload")
|
||||||
|
router.register("online/downloads", DownloadTaskViewSet, basename="download")
|
||||||
|
|
||||||
urlpatterns = [
|
urlpatterns = [
|
||||||
path("", include(router.urls)),
|
path("", include(router.urls)),
|
||||||
path(
|
path("online/file/", ClientDownloadView.as_view(), name="client_download"),
|
||||||
"online/downloads/",
|
|
||||||
DownloadToLibraryView.as_view(),
|
|
||||||
name="download_to_library",
|
|
||||||
),
|
|
||||||
]
|
]
|
||||||
|
|||||||
+106
-41
@@ -5,8 +5,10 @@ from urllib.parse import urlparse
|
|||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.core import signing
|
from django.core import signing
|
||||||
from django.db.models import Min
|
from django.db.models import Min
|
||||||
from django.http import Http404
|
from django.http import Http404, StreamingHttpResponse
|
||||||
from django.shortcuts import get_object_or_404
|
from django.shortcuts import get_object_or_404
|
||||||
|
from django.utils import timezone
|
||||||
|
from django.utils.text import get_valid_filename
|
||||||
from rest_framework import mixins, status, viewsets
|
from rest_framework import mixins, status, viewsets
|
||||||
from rest_framework.decorators import action
|
from rest_framework.decorators import action
|
||||||
from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
|
from rest_framework.permissions import AllowAny, IsAuthenticatedOrReadOnly
|
||||||
@@ -14,9 +16,10 @@ from rest_framework.response import Response
|
|||||||
from rest_framework.views import APIView
|
from rest_framework.views import APIView
|
||||||
|
|
||||||
from . import services
|
from . import services
|
||||||
from .models import MediaItem
|
from .downloads import start_download_task
|
||||||
|
from .models import DownloadTask, MediaItem
|
||||||
from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly
|
from .permissions import CanUpload, IsUploaderOrStaffOrReadOnly
|
||||||
from .serializers import MediaItemSerializer
|
from .serializers import DownloadTaskSerializer, MediaItemSerializer
|
||||||
|
|
||||||
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
LIST_ORDERINGS = {"name", "-name", "size", "-size", "created_at", "-created_at"}
|
||||||
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
MD5_RE = re.compile(r"[0-9a-fA-F]{32}")
|
||||||
@@ -188,59 +191,121 @@ class MediaItemViewSet(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class DownloadToLibraryView(APIView):
|
class DownloadTaskViewSet(
|
||||||
"""Fetch an e621 file into the watched folder and index it."""
|
mixins.ListModelMixin,
|
||||||
|
mixins.RetrieveModelMixin,
|
||||||
|
viewsets.GenericViewSet,
|
||||||
|
):
|
||||||
|
"""Background 'Download to Library' jobs."""
|
||||||
|
|
||||||
|
serializer_class = DownloadTaskSerializer
|
||||||
permission_classes = [CanUpload]
|
permission_classes = [CanUpload]
|
||||||
|
http_method_names = ["get", "post", "head", "options"]
|
||||||
|
|
||||||
def post(self, request):
|
def get_queryset(self):
|
||||||
|
queryset = DownloadTask.objects.select_related("library_item")
|
||||||
|
user = self.request.user
|
||||||
|
if not (user.is_staff or user.is_superuser):
|
||||||
|
queryset = queryset.filter(user=user)
|
||||||
|
return queryset
|
||||||
|
|
||||||
|
def create(self, request):
|
||||||
url = str(request.data.get("url") or "").strip()
|
url = str(request.data.get("url") or "").strip()
|
||||||
post_id = request.data.get("post_id")
|
post_id = request.data.get("post_id")
|
||||||
filename = str(request.data.get("filename") or "").strip()
|
filename = str(request.data.get("filename") or "").strip()
|
||||||
post_payload = request.data.get("post")
|
|
||||||
if not url.startswith(("http://", "https://")):
|
if not url.startswith(("http://", "https://")):
|
||||||
return Response(
|
return Response(
|
||||||
{"detail": "A valid file URL is required."},
|
{"detail": "A valid file URL is required."},
|
||||||
status=status.HTTP_400_BAD_REQUEST,
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
)
|
)
|
||||||
folder = Path(settings.WATCHED_FOLDER)
|
trimmed = services.trim_e621_post(request.data.get("post"))
|
||||||
name = (
|
resolved_post_id = None
|
||||||
filename
|
if trimmed is not None and trimmed.get("id") is not None:
|
||||||
or Path(urlparse(url).path).name
|
resolved_post_id = int(trimmed["id"])
|
||||||
or f"post-{post_id or 'download'}"
|
elif post_id is not None and str(post_id).isdigit():
|
||||||
|
resolved_post_id = int(post_id)
|
||||||
|
|
||||||
|
task = DownloadTask.objects.create(
|
||||||
|
user=request.user,
|
||||||
|
post_id=resolved_post_id,
|
||||||
|
url=url,
|
||||||
|
filename=filename,
|
||||||
|
e621_data=trimmed,
|
||||||
)
|
)
|
||||||
destination = services.unique_destination(folder, name)
|
start_download_task(task.id)
|
||||||
try:
|
|
||||||
services.download_file(url, destination)
|
|
||||||
item, _, location, _ = services.index_file(destination, folder)
|
|
||||||
except Exception as exc: # noqa: BLE001 - report download failures
|
|
||||||
destination.unlink(missing_ok=True)
|
|
||||||
return Response(
|
return Response(
|
||||||
{"detail": f"Download failed: {exc}"},
|
self.get_serializer(task).data, status=status.HTTP_202_ACCEPTED
|
||||||
|
)
|
||||||
|
|
||||||
|
@action(detail=True, methods=["post"])
|
||||||
|
def cancel(self, request, pk=None):
|
||||||
|
task = self.get_object()
|
||||||
|
if task.status not in {
|
||||||
|
DownloadTask.STATUS_PENDING,
|
||||||
|
DownloadTask.STATUS_DOWNLOADING,
|
||||||
|
}:
|
||||||
|
return Response(
|
||||||
|
{"detail": "Task is not running."},
|
||||||
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
|
)
|
||||||
|
DownloadTask.objects.filter(id=task.id).update(
|
||||||
|
cancelled=True, updated_at=timezone.now()
|
||||||
|
)
|
||||||
|
return Response({"success": True})
|
||||||
|
|
||||||
|
|
||||||
|
class ClientDownloadView(APIView):
|
||||||
|
"""Stream an e621 file straight to the browser (no library write)."""
|
||||||
|
|
||||||
|
permission_classes = [AllowAny]
|
||||||
|
|
||||||
|
def get(self, request):
|
||||||
|
url = str(request.query_params.get("url") or "").strip()
|
||||||
|
filename = str(request.query_params.get("filename") or "").strip()
|
||||||
|
parsed = urlparse(url)
|
||||||
|
if (
|
||||||
|
parsed.scheme not in {"http", "https"}
|
||||||
|
or parsed.hostname not in settings.E621_MEDIA_HOSTS
|
||||||
|
):
|
||||||
|
return Response(
|
||||||
|
{"detail": "URL not allowed."},
|
||||||
status=status.HTTP_400_BAD_REQUEST,
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
)
|
)
|
||||||
|
|
||||||
update_fields = []
|
import requests
|
||||||
if item.uploaded_by_id is None:
|
|
||||||
item.uploaded_by = request.user
|
|
||||||
update_fields.append("uploaded_by")
|
|
||||||
|
|
||||||
trimmed = services.trim_e621_post(post_payload)
|
try:
|
||||||
if trimmed is not None:
|
upstream = requests.get(
|
||||||
if trimmed.get("id") is not None:
|
url,
|
||||||
item.e621_post_id = int(trimmed["id"])
|
headers={"User-Agent": settings.USER_AGENT},
|
||||||
elif post_id and str(post_id).isdigit():
|
stream=True,
|
||||||
item.e621_post_id = int(post_id)
|
timeout=60,
|
||||||
item.e621_data = trimmed
|
|
||||||
update_fields += ["e621_post_id", "e621_data"]
|
|
||||||
if not item.rating and trimmed.get("rating") in {"s", "q", "e"}:
|
|
||||||
item.rating = trimmed["rating"]
|
|
||||||
update_fields.append("rating")
|
|
||||||
|
|
||||||
if update_fields:
|
|
||||||
item.save(update_fields=update_fields + ["updated_at"])
|
|
||||||
services.rename_location_to_j_id(item, location)
|
|
||||||
return Response(
|
|
||||||
MediaItemSerializer(item, context={"request": request}).data,
|
|
||||||
status=status.HTTP_201_CREATED,
|
|
||||||
)
|
)
|
||||||
|
upstream.raise_for_status()
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
return Response(
|
||||||
|
{"detail": f"Could not fetch the file: {exc}"},
|
||||||
|
status=status.HTTP_502_BAD_GATEWAY,
|
||||||
|
)
|
||||||
|
|
||||||
|
content_type = upstream.headers.get(
|
||||||
|
"Content-Type", "application/octet-stream"
|
||||||
|
)
|
||||||
|
name = get_valid_filename(
|
||||||
|
filename or Path(parsed.path).name or "download"
|
||||||
|
)
|
||||||
|
|
||||||
|
def stream():
|
||||||
|
try:
|
||||||
|
for chunk in upstream.iter_content(chunk_size=64 * 1024):
|
||||||
|
if chunk:
|
||||||
|
yield chunk
|
||||||
|
finally:
|
||||||
|
upstream.close()
|
||||||
|
|
||||||
|
response = StreamingHttpResponse(stream(), content_type=content_type)
|
||||||
|
response["Content-Disposition"] = f'attachment; filename="{name}"'
|
||||||
|
length = upstream.headers.get("Content-Length")
|
||||||
|
if length:
|
||||||
|
response["Content-Length"] = length
|
||||||
|
return response
|
||||||
|
|||||||
@@ -144,6 +144,15 @@ WATCHED_FOLDER = str(WATCHED_FOLDER)
|
|||||||
|
|
||||||
E621_BASE_URL = os.getenv("E621_BASE_URL", "https://e621.net").rstrip("/")
|
E621_BASE_URL = os.getenv("E621_BASE_URL", "https://e621.net").rstrip("/")
|
||||||
USER_AGENT = os.getenv("USER_AGENT", "J621/0.1 (by J621 on e621)")
|
USER_AGENT = os.getenv("USER_AGENT", "J621/0.1 (by J621 on e621)")
|
||||||
|
# Hosts the client-download proxy is allowed to stream from.
|
||||||
|
E621_MEDIA_HOSTS = [
|
||||||
|
host.strip()
|
||||||
|
for host in os.getenv(
|
||||||
|
"E621_MEDIA_HOSTS",
|
||||||
|
"static1.e621.net,static2.e621.net,static3.e621.net",
|
||||||
|
).split(",")
|
||||||
|
if host.strip()
|
||||||
|
]
|
||||||
|
|
||||||
# Guest visibility: e621's anonymous default blacklist is mirrored into the
|
# Guest visibility: e621's anonymous default blacklist is mirrored into the
|
||||||
# cache by `manage.py refresh_guest_blacklist`. This fallback is used until
|
# cache by `manage.py refresh_guest_blacklist`. This fallback is used until
|
||||||
|
|||||||
@@ -7,13 +7,13 @@ import {
|
|||||||
Heart,
|
Heart,
|
||||||
HeartOff,
|
HeartOff,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import { useMemo } from "react";
|
import { useMemo, useEffect, useState } from "react";
|
||||||
import { Link, useLocation, useNavigate } from "react-router-dom";
|
import { Link, useLocation, useNavigate } from "react-router-dom";
|
||||||
|
|
||||||
import { Button, EmptyState, Spinner, linkButtonClass } from "@/components/ui";
|
import { Button, EmptyState, Spinner, linkButtonClass } from "@/components/ui";
|
||||||
import { TagChip } from "@/components/TagChip";
|
import { TagChip } from "@/components/TagChip";
|
||||||
import { DetailShell } from "@/features/detail/DetailShell";
|
import { DetailShell } from "@/features/detail/DetailShell";
|
||||||
import { api } from "@/lib/api";
|
import { api, errorMessage } from "@/lib/api";
|
||||||
import { cn } from "@/lib/cn";
|
import { cn } from "@/lib/cn";
|
||||||
import { renderDText } from "@/lib/dtext";
|
import { renderDText } from "@/lib/dtext";
|
||||||
import {
|
import {
|
||||||
@@ -27,7 +27,7 @@ import {
|
|||||||
TAG_CATEGORY_ORDER,
|
TAG_CATEGORY_ORDER,
|
||||||
} from "@/lib/e621";
|
} from "@/lib/e621";
|
||||||
import { formatBytes, formatDate } from "@/lib/format";
|
import { formatBytes, formatDate } from "@/lib/format";
|
||||||
import type { MediaItem } from "@/lib/types";
|
import type { DownloadTask } from "@/lib/types";
|
||||||
import { useAuth } from "@/store/auth";
|
import { useAuth } from "@/store/auth";
|
||||||
import { useE621 } from "@/store/e621";
|
import { useE621 } from "@/store/e621";
|
||||||
|
|
||||||
@@ -116,9 +116,11 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const [downloadTaskId, setDownloadTaskId] = useState<string | null>(null);
|
||||||
|
|
||||||
const downloadMutation = useMutation({
|
const downloadMutation = useMutation({
|
||||||
mutationFn: (target: NonNullable<typeof post>) =>
|
mutationFn: (target: NonNullable<typeof post>) =>
|
||||||
api<MediaItem>("/api/online/downloads/", {
|
api<DownloadTask>("/api/online/downloads/", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
json: {
|
json: {
|
||||||
post_id: target.id,
|
post_id: target.id,
|
||||||
@@ -127,12 +129,36 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
post: target,
|
post: target,
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
onSuccess: (item) => {
|
onSuccess: (task) => setDownloadTaskId(task.task_id),
|
||||||
void queryClient.invalidateQueries({ queryKey: ["library-lookup"] });
|
});
|
||||||
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
|
||||||
navigate(`/detail/${item.j_id}`);
|
const taskQuery = useQuery({
|
||||||
|
queryKey: ["download-task", downloadTaskId],
|
||||||
|
queryFn: () =>
|
||||||
|
api<DownloadTask>(`/api/online/downloads/${downloadTaskId}/`),
|
||||||
|
enabled: Boolean(downloadTaskId),
|
||||||
|
refetchInterval: (query) => {
|
||||||
|
const status = query.state.data?.status;
|
||||||
|
return status === "pending" || status === "downloading" ? 700 : false;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
const downloadTask = taskQuery.data;
|
||||||
|
const downloadActive =
|
||||||
|
downloadTask?.status === "pending" || downloadTask?.status === "downloading";
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (downloadTask?.status === "complete") {
|
||||||
|
void queryClient.invalidateQueries({ queryKey: ["library-lookup"] });
|
||||||
|
void queryClient.invalidateQueries({ queryKey: ["files"] });
|
||||||
|
}
|
||||||
|
}, [downloadTask?.status, queryClient]);
|
||||||
|
|
||||||
|
function cancelDownload() {
|
||||||
|
if (!downloadTaskId) return;
|
||||||
|
void api(`/api/online/downloads/${downloadTaskId}/cancel/`, {
|
||||||
|
method: "POST",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if (query.isPending) {
|
if (query.isPending) {
|
||||||
return (
|
return (
|
||||||
@@ -163,6 +189,11 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
const imageSrc =
|
const imageSrc =
|
||||||
post.sample.has && post.sample.url ? post.sample.url : post.file.url;
|
post.sample.has && post.sample.url ? post.sample.url : post.file.url;
|
||||||
const canUpload = Boolean(user?.can_upload);
|
const canUpload = Boolean(user?.can_upload);
|
||||||
|
const clientDownloadUrl = post.file.url
|
||||||
|
? `/api/online/file/?url=${encodeURIComponent(post.file.url)}&filename=${encodeURIComponent(
|
||||||
|
`${post.file.md5}.${post.file.ext}`,
|
||||||
|
)}`
|
||||||
|
: null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<DetailShell
|
<DetailShell
|
||||||
@@ -213,7 +244,10 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
<Button
|
<Button
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
disabled={
|
disabled={
|
||||||
!canUpload || downloadMutation.isPending || !post.file.url
|
!canUpload ||
|
||||||
|
downloadMutation.isPending ||
|
||||||
|
downloadActive ||
|
||||||
|
!post.file.url
|
||||||
}
|
}
|
||||||
onClick={() => downloadMutation.mutate(post)}
|
onClick={() => downloadMutation.mutate(post)}
|
||||||
title={
|
title={
|
||||||
@@ -223,12 +257,22 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
}
|
}
|
||||||
>
|
>
|
||||||
<Download className="h-4 w-4" />
|
<Download className="h-4 w-4" />
|
||||||
{downloadMutation.isPending
|
{downloadMutation.isPending || downloadActive
|
||||||
? "Downloading…"
|
? "Downloading…"
|
||||||
: "Download to Library"}
|
: "Download to Library"}
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{clientDownloadUrl ? (
|
||||||
|
<a
|
||||||
|
href={clientDownloadUrl}
|
||||||
|
className={linkButtonClass}
|
||||||
|
title="Save the original file straight to your computer"
|
||||||
|
>
|
||||||
|
<Download className="h-4 w-4" /> Download to client
|
||||||
|
</a>
|
||||||
|
) : null}
|
||||||
|
|
||||||
<a
|
<a
|
||||||
href={`https://e621.net/posts/${post.id}`}
|
href={`https://e621.net/posts/${post.id}`}
|
||||||
target="_blank"
|
target="_blank"
|
||||||
@@ -239,6 +283,59 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{downloadTask &&
|
||||||
|
(downloadTask.status === "pending" ||
|
||||||
|
downloadTask.status === "downloading") ? (
|
||||||
|
<div className="rounded-lg border border-ctp-surface0 bg-ctp-base p-3">
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-3 text-xs">
|
||||||
|
<span className="text-ctp-subtext1">
|
||||||
|
Downloading to the library… {downloadTask.progress}%
|
||||||
|
</span>
|
||||||
|
<span className="font-mono text-ctp-overlay0">
|
||||||
|
{formatBytes(downloadTask.downloaded)}
|
||||||
|
{downloadTask.total > 0
|
||||||
|
? ` / ${formatBytes(downloadTask.total)}`
|
||||||
|
: ""}
|
||||||
|
{downloadTask.speed
|
||||||
|
? ` · ${(downloadTask.speed / 1024 / 1024).toFixed(1)} MB/s`
|
||||||
|
: ""}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div className="mt-2 h-1.5 w-full overflow-hidden rounded-full bg-ctp-surface0">
|
||||||
|
<div
|
||||||
|
className="h-full bg-ctp-mauve transition-[width]"
|
||||||
|
style={{ width: `${downloadTask.progress}%` }}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={cancelDownload}
|
||||||
|
className="mt-2 rounded-md px-1.5 py-0.5 font-mono text-[10px] text-ctp-overlay0 transition hover:bg-ctp-surface0 hover:text-ctp-text"
|
||||||
|
>
|
||||||
|
cancel
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
) : downloadTask?.status === "complete" ? (
|
||||||
|
<p className="flex items-center gap-2 rounded-md border border-ctp-green/30 bg-ctp-green/10 px-3 py-2 text-sm text-ctp-green">
|
||||||
|
<CheckCircle2 className="h-4 w-4 shrink-0" />
|
||||||
|
Downloaded to the library
|
||||||
|
{downloadTask.library_j_id ? (
|
||||||
|
<Link
|
||||||
|
to={`/detail/${downloadTask.library_j_id}`}
|
||||||
|
className="font-mono text-ctp-blue hover:underline"
|
||||||
|
>
|
||||||
|
{downloadTask.library_j_id}
|
||||||
|
</Link>
|
||||||
|
) : null}
|
||||||
|
</p>
|
||||||
|
) : downloadTask?.status === "cancelled" ? (
|
||||||
|
<p className="text-xs text-ctp-overlay0">Download cancelled.</p>
|
||||||
|
) : downloadTask?.status === "error" ? (
|
||||||
|
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
||||||
|
{downloadTask.error || "Download failed."}
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
|
|
||||||
{favoriteMutation.isError ? (
|
{favoriteMutation.isError ? (
|
||||||
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
||||||
{e621ErrorMessage(favoriteMutation.error)}
|
{e621ErrorMessage(favoriteMutation.error)}
|
||||||
@@ -246,7 +343,7 @@ export function OnlineDetail({ postId }: { postId: number }) {
|
|||||||
) : null}
|
) : null}
|
||||||
{downloadMutation.isError ? (
|
{downloadMutation.isError ? (
|
||||||
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
<p className="rounded-md border border-ctp-red/30 bg-ctp-red/10 px-3 py-2 text-sm text-ctp-red">
|
||||||
{e621ErrorMessage(downloadMutation.error)}
|
{errorMessage(downloadMutation.error)}
|
||||||
</p>
|
</p>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
|
|||||||
@@ -107,6 +107,21 @@ export interface FileNeighbors {
|
|||||||
next: FileNeighbor | null;
|
next: FileNeighbor | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DownloadTask {
|
||||||
|
task_id: string;
|
||||||
|
post_id: number | null;
|
||||||
|
filename: string;
|
||||||
|
status: "pending" | "downloading" | "complete" | "error" | "cancelled";
|
||||||
|
progress: number;
|
||||||
|
downloaded: number;
|
||||||
|
total: number;
|
||||||
|
speed: number | null;
|
||||||
|
error: string;
|
||||||
|
library_j_id: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface E621IqdbCandidate {
|
export interface E621IqdbCandidate {
|
||||||
post_id: number | null;
|
post_id: number | null;
|
||||||
score: number | null;
|
score: number | null;
|
||||||
|
|||||||
Reference in New Issue
Block a user