Add job cancellation to the stats dashboard
- Active jobs on /stats get a cancel button wired to the existing download/match cancel endpoints, showing "cancelling..." and an inline error when the task already finished. - Cancelling now sets the status immediately, so a task whose runner died in a restart stops showing as "downloading". - Download streams use a bounded read timeout (10 s connect / 60 s read): a stalled socket fails within a minute (previously it could block forever), and a task cancelled while stalled is marked cancelled rather than error. - The stats job list reaps stale download/match tasks, so phantom jobs never appear on the dashboard.
This commit is contained in:
@@ -122,6 +122,7 @@ def _download_job(task):
|
|||||||
"downloaded": task.downloaded,
|
"downloaded": task.downloaded,
|
||||||
"total": task.total,
|
"total": task.total,
|
||||||
"detail": detail,
|
"detail": detail,
|
||||||
|
"cancelled": task.cancelled,
|
||||||
"created_at": task.created_at,
|
"created_at": task.created_at,
|
||||||
"updated_at": task.updated_at,
|
"updated_at": task.updated_at,
|
||||||
}
|
}
|
||||||
@@ -143,6 +144,7 @@ def _match_job(task):
|
|||||||
f"{task.processed}/{task.total} · {task.matched} matched · "
|
f"{task.processed}/{task.total} · {task.matched} matched · "
|
||||||
f"{task.not_found} not found · {task.deleted} deleted"
|
f"{task.not_found} not found · {task.deleted} deleted"
|
||||||
),
|
),
|
||||||
|
"cancelled": task.cancelled,
|
||||||
"created_at": task.created_at,
|
"created_at": task.created_at,
|
||||||
"updated_at": task.updated_at,
|
"updated_at": task.updated_at,
|
||||||
}
|
}
|
||||||
@@ -150,8 +152,14 @@ def _match_job(task):
|
|||||||
|
|
||||||
def jobs_stats():
|
def jobs_stats():
|
||||||
"""Running/pending jobs plus the most recently finished ones."""
|
"""Running/pending jobs plus the most recently finished ones."""
|
||||||
|
from apps.library.downloads import reap_stale_downloads
|
||||||
|
from apps.library.matching import reap_stale_match_tasks
|
||||||
from apps.library.models import DownloadTask, MatchTask
|
from apps.library.models import DownloadTask, MatchTask
|
||||||
|
|
||||||
|
# Tasks whose runner died in a restart must not linger as "downloading".
|
||||||
|
reap_stale_downloads()
|
||||||
|
reap_stale_match_tasks()
|
||||||
|
|
||||||
active_statuses_download = [
|
active_statuses_download = [
|
||||||
DownloadTask.STATUS_PENDING,
|
DownloadTask.STATUS_PENDING,
|
||||||
DownloadTask.STATUS_DOWNLOADING,
|
DownloadTask.STATUS_DOWNLOADING,
|
||||||
|
|||||||
@@ -144,6 +144,18 @@ def run_download_task(task_id):
|
|||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001 - report background failures
|
except Exception as exc: # noqa: BLE001 - report background failures
|
||||||
destination.unlink(missing_ok=True)
|
destination.unlink(missing_ok=True)
|
||||||
|
if DownloadTask.objects.filter(id=task_id, cancelled=True).exists():
|
||||||
|
# Cancel was requested while the socket was stalled; the read
|
||||||
|
# timeout is what breaks the worker out of it.
|
||||||
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
|
status=DownloadTask.STATUS_CANCELLED,
|
||||||
|
progress=0,
|
||||||
|
downloaded=0,
|
||||||
|
speed=None,
|
||||||
|
error="",
|
||||||
|
updated_at=timezone.now(),
|
||||||
|
)
|
||||||
|
else:
|
||||||
logger.exception("Download task %s failed", task_id)
|
logger.exception("Download task %s failed", task_id)
|
||||||
DownloadTask.objects.filter(id=task_id).update(
|
DownloadTask.objects.filter(id=task_id).update(
|
||||||
status=DownloadTask.STATUS_ERROR,
|
status=DownloadTask.STATUS_ERROR,
|
||||||
|
|||||||
@@ -250,13 +250,24 @@ class DownloadCancelled(Exception):
|
|||||||
|
|
||||||
|
|
||||||
def download_file(
|
def download_file(
|
||||||
url, destination, progress_callback=None, should_cancel=None
|
url,
|
||||||
|
destination,
|
||||||
|
progress_callback=None,
|
||||||
|
should_cancel=None,
|
||||||
|
read_timeout=60,
|
||||||
):
|
):
|
||||||
"""Stream a remote file into destination (used by Download to Library)."""
|
"""Stream a remote file into destination (used by Download to Library).
|
||||||
|
|
||||||
|
The read timeout bounds how long a stalled connection can block the
|
||||||
|
worker: without it a hung socket would keep a job "downloading" forever
|
||||||
|
and the cancel flag could never be observed.
|
||||||
|
"""
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
headers = {"User-Agent": settings.USER_AGENT}
|
headers = {"User-Agent": settings.USER_AGENT}
|
||||||
with requests.get(url, headers=headers, stream=True, timeout=120) as response:
|
with requests.get(
|
||||||
|
url, headers=headers, stream=True, timeout=(10, read_timeout)
|
||||||
|
) as response:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
total = int(response.headers.get("content-length") or 0)
|
total = int(response.headers.get("content-length") or 0)
|
||||||
downloaded = 0
|
downloaded = 0
|
||||||
|
|||||||
@@ -465,7 +465,10 @@ class DownloadTaskViewSet(
|
|||||||
status=status.HTTP_400_BAD_REQUEST,
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
)
|
)
|
||||||
DownloadTask.objects.filter(id=task.id).update(
|
DownloadTask.objects.filter(id=task.id).update(
|
||||||
cancelled=True, updated_at=timezone.now()
|
cancelled=True,
|
||||||
|
status=DownloadTask.STATUS_CANCELLED,
|
||||||
|
speed=None,
|
||||||
|
updated_at=timezone.now(),
|
||||||
)
|
)
|
||||||
return Response({"success": True})
|
return Response({"success": True})
|
||||||
|
|
||||||
@@ -508,7 +511,9 @@ class MatchTaskViewSet(
|
|||||||
status=status.HTTP_400_BAD_REQUEST,
|
status=status.HTTP_400_BAD_REQUEST,
|
||||||
)
|
)
|
||||||
MatchTask.objects.filter(id=task.id).update(
|
MatchTask.objects.filter(id=task.id).update(
|
||||||
cancelled=True, updated_at=timezone.now()
|
cancelled=True,
|
||||||
|
status=MatchTask.STATUS_CANCELLED,
|
||||||
|
updated_at=timezone.now(),
|
||||||
)
|
)
|
||||||
return Response({"success": True})
|
return Response({"success": True})
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import { useQuery } from "@tanstack/react-query";
|
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||||
import { Activity, Download, Gauge, HardDrive, MemoryStick, ScanSearch, Terminal } from "lucide-react";
|
import { Activity, Download, Gauge, HardDrive, MemoryStick, ScanSearch, Terminal } from "lucide-react";
|
||||||
import { useEffect, useRef, useState, type ReactNode } from "react";
|
import { useEffect, useRef, useState, type ReactNode } from "react";
|
||||||
|
|
||||||
import { EmptyState, Spinner } from "@/components/ui";
|
import { EmptyState, Spinner } from "@/components/ui";
|
||||||
import { api } from "@/lib/api";
|
import { api, errorMessage } from "@/lib/api";
|
||||||
import { cn } from "@/lib/cn";
|
import { cn } from "@/lib/cn";
|
||||||
import { formatBytes } from "@/lib/format";
|
import { formatBytes } from "@/lib/format";
|
||||||
import type { StatJob, SystemStats } from "@/lib/types";
|
import type { StatJob, SystemStats } from "@/lib/types";
|
||||||
@@ -77,7 +77,19 @@ function jobStatusClass(status: string): string {
|
|||||||
return "text-ctp-blue";
|
return "text-ctp-blue";
|
||||||
}
|
}
|
||||||
|
|
||||||
function JobRow({ job, showSummary }: { job: StatJob; showSummary?: boolean }) {
|
function JobRow({
|
||||||
|
job,
|
||||||
|
showSummary,
|
||||||
|
onCancel,
|
||||||
|
cancelling,
|
||||||
|
cancelError,
|
||||||
|
}: {
|
||||||
|
job: StatJob;
|
||||||
|
showSummary?: boolean;
|
||||||
|
onCancel?: () => void;
|
||||||
|
cancelling?: boolean;
|
||||||
|
cancelError?: string | null;
|
||||||
|
}) {
|
||||||
const Icon = job.kind === "download" ? Download : ScanSearch;
|
const Icon = job.kind === "download" ? Download : ScanSearch;
|
||||||
return (
|
return (
|
||||||
<li className="flex flex-col gap-1">
|
<li className="flex flex-col gap-1">
|
||||||
@@ -87,7 +99,21 @@ function JobRow({ job, showSummary }: { job: StatJob; showSummary?: boolean }) {
|
|||||||
<span className={cn("ml-auto shrink-0 font-mono text-[11px]", jobStatusClass(job.status))}>
|
<span className={cn("ml-auto shrink-0 font-mono text-[11px]", jobStatusClass(job.status))}>
|
||||||
{job.status}
|
{job.status}
|
||||||
</span>
|
</span>
|
||||||
|
{onCancel ? (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onCancel}
|
||||||
|
disabled={cancelling || job.cancelled}
|
||||||
|
className="shrink-0 rounded border border-ctp-red/40 px-1.5 py-0.5 font-mono text-[10px] text-ctp-red transition hover:bg-ctp-red/15 disabled:opacity-50"
|
||||||
|
title="Cancel this job"
|
||||||
|
>
|
||||||
|
{job.cancelled || cancelling ? "cancelling…" : "cancel"}
|
||||||
|
</button>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
|
{cancelError ? (
|
||||||
|
<p className="pl-5 text-[11px] text-ctp-red">{cancelError}</p>
|
||||||
|
) : null}
|
||||||
{showSummary ? (
|
{showSummary ? (
|
||||||
<p className="truncate pl-5 text-[11px] text-ctp-overlay0" title={job.summary ?? job.detail}>
|
<p className="truncate pl-5 text-[11px] text-ctp-overlay0" title={job.summary ?? job.detail}>
|
||||||
{job.summary ?? job.detail}
|
{job.summary ?? job.detail}
|
||||||
@@ -114,6 +140,7 @@ function logLineClass(line: string): string {
|
|||||||
|
|
||||||
export default function StatsPage() {
|
export default function StatsPage() {
|
||||||
const user = useAuth((state) => state.user);
|
const user = useAuth((state) => state.user);
|
||||||
|
const queryClient = useQueryClient();
|
||||||
const staff = Boolean(
|
const staff = Boolean(
|
||||||
user &&
|
user &&
|
||||||
(user.is_staff || user.is_superuser || user.role === "staff"),
|
(user.is_staff || user.is_superuser || user.role === "staff"),
|
||||||
@@ -124,6 +151,19 @@ export default function StatsPage() {
|
|||||||
enabled: staff,
|
enabled: staff,
|
||||||
refetchInterval: staff ? REFRESH_MS : false,
|
refetchInterval: staff ? REFRESH_MS : false,
|
||||||
});
|
});
|
||||||
|
const cancelMutation = useMutation({
|
||||||
|
mutationFn: (job: StatJob) =>
|
||||||
|
api(
|
||||||
|
job.kind === "download"
|
||||||
|
? `/api/online/downloads/${job.id}/cancel/`
|
||||||
|
: `/api/matches/${job.id}/cancel/`,
|
||||||
|
{ method: "POST" },
|
||||||
|
),
|
||||||
|
onSuccess: () => {
|
||||||
|
void queryClient.invalidateQueries({ queryKey: ["stats"] });
|
||||||
|
void queryClient.invalidateQueries({ queryKey: ["status"] });
|
||||||
|
},
|
||||||
|
});
|
||||||
const [autoScroll, setAutoScroll] = useState(true);
|
const [autoScroll, setAutoScroll] = useState(true);
|
||||||
const logRef = useRef<HTMLDivElement>(null);
|
const logRef = useRef<HTMLDivElement>(null);
|
||||||
const data = query.data;
|
const data = query.data;
|
||||||
@@ -321,7 +361,21 @@ export default function StatsPage() {
|
|||||||
) : (
|
) : (
|
||||||
<ul className="flex flex-col gap-3">
|
<ul className="flex flex-col gap-3">
|
||||||
{jobs.active.map((job) => (
|
{jobs.active.map((job) => (
|
||||||
<JobRow key={`${job.kind}-${job.id}`} job={job} />
|
<JobRow
|
||||||
|
key={`${job.kind}-${job.id}`}
|
||||||
|
job={job}
|
||||||
|
onCancel={() => cancelMutation.mutate(job)}
|
||||||
|
cancelling={
|
||||||
|
cancelMutation.isPending &&
|
||||||
|
cancelMutation.variables?.id === job.id
|
||||||
|
}
|
||||||
|
cancelError={
|
||||||
|
cancelMutation.isError &&
|
||||||
|
cancelMutation.variables?.id === job.id
|
||||||
|
? errorMessage(cancelMutation.error)
|
||||||
|
: null
|
||||||
|
}
|
||||||
|
/>
|
||||||
))}
|
))}
|
||||||
</ul>
|
</ul>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -338,6 +338,7 @@ export interface StatJob {
|
|||||||
progress: number;
|
progress: number;
|
||||||
detail: string;
|
detail: string;
|
||||||
summary?: string;
|
summary?: string;
|
||||||
|
cancelled?: boolean;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user