Add the staff stats dashboard

Backend: GET /api/stats/ (staff only) gathers psutil CPU/memory counters,
nvidia-smi GPU stats, the cached disk numbers and the running/finished
download + match jobs. Root logging now also writes a rotating file
(backend/logs/j621.log) so the dashboard can tail it, and psutil joins the
requirements. The storage payload computation is shared with the existing
storage endpoint.

Frontend: a /stats route + Stats nav entry for staff, polling every 2 s —
per-core CPU bars, memory and swap, GPUs (utilization, VRAM, temperature),
disk with the media/temp breakdown, active jobs with progress bars,
recently finished jobs with summaries, and the log tail with level colours
and an auto-scroll toggle. Section 4 of the roadmap is complete.

backend/apps/core/stats.py

@@ -0,0 +1,237 @@
+"""Runtime statistics for the staff dashboard.
+
+Everything here is read-only and cheap enough to poll every couple of
+seconds: psutil counters, an optional nvidia-smi query, the cached storage
+numbers and the tail of the application log file.
+"""
+
+import logging
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import psutil
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+NVIDIA_QUERY = (
+    "name,utilization.gpu,memory.used,memory.total,temperature.gpu"
+)
+LOG_TAIL_LINES = 140
+RECENT_LIMIT = 6
+
+# Prime the non-blocking CPU counters so the first request already has a delta.
+psutil.cpu_percent(interval=None)
+psutil.cpu_percent(interval=None, percpu=True)
+
+
+def _as_int(value):
+    try:
+        return int(float(str(value).strip()))
+    except (TypeError, ValueError):
+        return None
+
+
+def system_stats():
+    """CPU, memory and uptime counters (non-blocking)."""
+    memory = psutil.virtual_memory()
+    swap = psutil.swap_memory()
+    try:
+        load = [round(value, 2) for value in psutil.getloadavg()]
+    except (AttributeError, OSError):
+        load = []
+    return {
+        "cpu": {
+            "percent": psutil.cpu_percent(interval=None),
+            "per_cpu": psutil.cpu_percent(interval=None, percpu=True),
+            "cores": psutil.cpu_count(logical=True),
+            "physical_cores": psutil.cpu_count(logical=False),
+            "load": load,
+        },
+        "memory": {
+            "total": memory.total,
+            "used": memory.used,
+            "available": memory.available,
+            "percent": memory.percent,
+            "swap_total": swap.total,
+            "swap_used": swap.used,
+            "swap_percent": swap.percent,
+        },
+        "uptime_seconds": int(time.time() - psutil.boot_time()),
+    }
+
+
+def gpu_stats():
+    """NVIDIA GPUs via nvidia-smi, or an empty list when unavailable."""
+    try:
+        result = subprocess.run(
+            [
+                "nvidia-smi",
+                f"--query-gpu={NVIDIA_QUERY}",
+                "--format=csv,noheader,nounits",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=4,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        logger.debug("nvidia-smi unavailable: %s", exc)
+        return []
+    if result.returncode != 0:
+        return []
+    gpus = []
+    for line in result.stdout.strip().splitlines():
+        parts = [part.strip() for part in line.split(",")]
+        if len(parts) != 5:
+            continue
+        name, utilization, memory_used, memory_total, temperature = parts
+        used_mib = _as_int(memory_used)
+        total_mib = _as_int(memory_total)
+        gpus.append(
+            {
+                "name": name,
+                "utilization": _as_int(utilization),
+                "memory_used": used_mib * 1024 * 1024
+                if used_mib is not None
+                else None,
+                "memory_total": total_mib * 1024 * 1024
+                if total_mib is not None
+                else None,
+                "temperature": _as_int(temperature),
+            }
+        )
+    return gpus
+
+
+def _download_job(task):
+    label = (
+        f"Post #{task.post_id}"
+        if task.post_id
+        else (task.filename or "Download")
+    )
+    detail = f"{task.downloaded} / {task.total}" if task.total else task.filename
+    return {
+        "kind": "download",
+        "id": str(task.id),
+        "label": label,
+        "status": task.status,
+        "progress": task.progress,
+        "downloaded": task.downloaded,
+        "total": task.total,
+        "detail": detail,
+        "created_at": task.created_at,
+        "updated_at": task.updated_at,
+    }
+
+
+def _match_job(task):
+    progress = (
+        int(task.processed * 100 / task.total) if task.total else 0
+    )
+    return {
+        "kind": "match",
+        "id": str(task.id),
+        "label": f"e621 match scan ({task.scope})",
+        "status": task.status,
+        "progress": progress,
+        "processed": task.processed,
+        "total": task.total,
+        "detail": (
+            f"{task.processed}/{task.total} · {task.matched} matched · "
+            f"{task.not_found} not found · {task.deleted} deleted"
+        ),
+        "created_at": task.created_at,
+        "updated_at": task.updated_at,
+    }
+
+
+def jobs_stats():
+    """Running/pending jobs plus the most recently finished ones."""
+    from apps.library.models import DownloadTask, MatchTask
+
+    active_statuses_download = [
+        DownloadTask.STATUS_PENDING,
+        DownloadTask.STATUS_DOWNLOADING,
+    ]
+    active_statuses_match = [
+        MatchTask.STATUS_PENDING,
+        MatchTask.STATUS_RUNNING,
+    ]
+    finished_download = [
+        DownloadTask.STATUS_COMPLETE,
+        DownloadTask.STATUS_ERROR,
+        DownloadTask.STATUS_CANCELLED,
+    ]
+    finished_match = [
+        MatchTask.STATUS_COMPLETE,
+        MatchTask.STATUS_ERROR,
+        MatchTask.STATUS_CANCELLED,
+    ]
+
+    active = [
+        _download_job(task)
+        for task in DownloadTask.objects.filter(
+            status__in=active_statuses_download
+        ).order_by("created_at")[:RECENT_LIMIT]
+    ]
+    active += [
+        _match_job(task)
+        for task in MatchTask.objects.filter(
+            status__in=active_statuses_match
+        ).order_by("created_at")[:RECENT_LIMIT]
+    ]
+
+    recent = []
+    for task in DownloadTask.objects.filter(
+        status__in=finished_download
+    ).order_by("-updated_at")[:RECENT_LIMIT]:
+        entry = _download_job(task)
+        entry["summary"] = (
+            f"J-{task.library_item_id}"
+            if task.library_item_id
+            else (task.error[:160] if task.error else task.status)
+        )
+        recent.append(entry)
+    for task in MatchTask.objects.filter(
+        status__in=finished_match
+    ).order_by("-updated_at")[:RECENT_LIMIT]:
+        entry = _match_job(task)
+        entry["summary"] = task.error[:160] if task.error else entry["detail"]
+        recent.append(entry)
+    recent.sort(key=lambda entry: entry["updated_at"], reverse=True)
+
+    return {
+        "active": active,
+        "recent": recent[:RECENT_LIMIT],
+        "active_count": len(active),
+    }
+
+
+def tail_lines(path, count=LOG_TAIL_LINES):
+    """Last `count` lines of a file without reading all of it."""
+    with open(path, "rb") as handle:
+        handle.seek(0, os.SEEK_END)
+        size = handle.tell()
+        block = 16384
+        data = b""
+        while size > 0 and data.count(b"\n") <= count:
+            step = min(block, size)
+            size -= step
+            handle.seek(size)
+            data = handle.read(step) + data
+    return data.decode("utf-8", errors="replace").splitlines()[-count:]
+
+
+def log_tail():
+    path = Path(getattr(settings, "LOG_FILE", ""))
+    if not path or not path.exists():
+        return {"path": str(path), "lines": [], "missing": True}
+    try:
+        lines = tail_lines(path)
+    except OSError as exc:
+        logger.warning("Could not read the log file: %s", exc)
+        return {"path": str(path), "lines": [], "missing": True}
+    return {"path": str(path), "lines": lines, "missing": False}

backend/apps/core/urls.py

@@ -1,7 +1,8 @@
 from django.urls import path
 
-from .views import StatusView
+from .views import StatsView, StatusView
 
 urlpatterns = [
     path("status/", StatusView.as_view(), name="status"),
+    path("stats/", StatsView.as_view(), name="stats"),
 ]

backend/apps/core/views.py

@@ -1,10 +1,12 @@
 import time
 
 from django.conf import settings
-from rest_framework.permissions import AllowAny
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
+from . import stats
 from .system_info import get_os_info
 
 
@@ -40,3 +42,27 @@ class StatusView(APIView):
         }
         payload["server_time_ms"] = round((time.perf_counter() - started) * 1000, 1)
         return Response(payload)
+
+
+class StatsView(APIView):
+    """Staff-only runtime statistics for the dashboard."""
+
+    permission_classes = [IsAuthenticated]
+
+    def get(self, request):
+        user = request.user
+        if not (
+            user.is_staff or user.is_superuser or user.role == user.ROLE_STAFF
+        ):
+            raise PermissionDenied("Staff only.")
+        from apps.library.tools import storage_info
+
+        return Response(
+            {
+                "system": stats.system_stats(),
+                "gpus": stats.gpu_stats(),
+                "disk": storage_info(),
+                "jobs": stats.jobs_stats(),
+                "log": stats.log_tail(),
+            }
+        )

backend/apps/library/tools.py

@@ -395,57 +395,62 @@ class TagCloudView(APIView):
         return Response(payload)
 
 
+def storage_info():
+    """Disk usage payload, cached briefly (also used by the stats dashboard)."""
+    cached = cache.get(STORAGE_CACHE_KEY)
+    if cached is not None:
+        return cached
+
+    watched = Path(settings.WATCHED_FOLDER)
+    media_root = Path(settings.MEDIA_ROOT)
+    temp_dir = media_root / "uploads" / "temp"
+
+    usage = shutil.disk_usage(
+        str(watched) if watched.exists() else str(Path(settings.BASE_DIR))
+    )
+    library_size = (
+        sum(file.stat().st_size for file in watched.rglob("*") if file.is_file())
+        if watched.exists()
+        else 0
+    )
+    media_size = (
+        sum(file.stat().st_size for file in media_root.rglob("*") if file.is_file())
+        if media_root.exists()
+        else 0
+    )
+    temp_files = (
+        [file for file in temp_dir.rglob("*") if file.is_file()]
+        if temp_dir.exists()
+        else []
+    )
+
+    payload = {
+        "watched_folder": {
+            "path": str(watched),
+            "total": usage.total,
+            "used": usage.used,
+            "free": usage.free,
+            "library_size": library_size,
+            "percent_used": (
+                round(usage.used / usage.total * 100, 1) if usage.total else 0
+            ),
+        },
+        "media": {"path": str(media_root), "size": media_size},
+        "temp": {
+            "path": str(temp_dir),
+            "size": sum(file.stat().st_size for file in temp_files),
+            "files": len(temp_files),
+        },
+        "library_items": MediaItem.objects.count(),
+    }
+    cache.set(STORAGE_CACHE_KEY, payload, STORAGE_CACHE_TTL)
+    return payload
+
+
 class StorageView(APIView):
     """Disk usage for the watched folder, media root and temp uploads."""
 
     permission_classes = [IsAuthenticated]
 
     def get(self, request):
-        cached = cache.get(STORAGE_CACHE_KEY)
-        if cached is not None:
-            return Response(cached)
-
-        watched = Path(settings.WATCHED_FOLDER)
-        media_root = Path(settings.MEDIA_ROOT)
-        temp_dir = media_root / "uploads" / "temp"
-
-        usage = shutil.disk_usage(
-            str(watched) if watched.exists() else str(Path(settings.BASE_DIR))
-        )
-        library_size = (
-            sum(file.stat().st_size for file in watched.rglob("*") if file.is_file())
-            if watched.exists()
-            else 0
-        )
-        media_size = (
-            sum(file.stat().st_size for file in media_root.rglob("*") if file.is_file())
-            if media_root.exists()
-            else 0
-        )
-        temp_files = (
-            [file for file in temp_dir.rglob("*") if file.is_file()]
-            if temp_dir.exists()
-            else []
-        )
-
-        payload = {
-            "watched_folder": {
-                "path": str(watched),
-                "total": usage.total,
-                "used": usage.used,
-                "free": usage.free,
-                "library_size": library_size,
-                "percent_used": (
-                    round(usage.used / usage.total * 100, 1) if usage.total else 0
-                ),
-            },
-            "media": {"path": str(media_root), "size": media_size},
-            "temp": {
-                "path": str(temp_dir),
-                "size": sum(file.stat().st_size for file in temp_files),
-                "files": len(temp_files),
-            },
-            "library_items": MediaItem.objects.count(),
-        }
-        cache.set(STORAGE_CACHE_KEY, payload, STORAGE_CACHE_TTL)
-        return Response(payload)
+        return Response(storage_info())