finished stats page

2026-08-05 14:17:41 -05:00
parent 6325c67790
commit 2d73638a06
5 changed files with 225 additions and 6 deletions
@@ -8,3 +8,4 @@ nonpacks/media/
 nonpacks/static/vanilla/entity_index.json
 *.log
 AGENTS/
+logs/
@@ -1,3 +1,8 @@
+import tempfile
+import time
+import unittest.mock
+from pathlib import Path
+
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 from django.test import TestCase
@@ -26,7 +31,7 @@ class ServerStatsTests(TestCase):
        resp = self.client.get(reverse('landing:server_stats_api'))
        self.assertEqual(resp.status_code, 200)
        data = resp.json()
-        for key in ('app', 'system', 'content', 'recent', 'generated_at'):
+        for key in ('app', 'system', 'content', 'recent', 'traffic', 'generated_at'):
            self.assertIn(key, data)
        self.assertIn('users', data['content'])
        self.assertIn('versions', data['recent'])
@@ -59,3 +64,33 @@ class ServerStatsTests(TestCase):
        self._authorized()
        resp = self.client.get(reverse('landing:server_stats'))
        self.assertEqual(resp.status_code, 302)
+
+
+class TrafficStatsTests(TestCase):
+    def test_parses_access_log(self):
+        from landing import views
+        with tempfile.TemporaryDirectory() as tmp:
+            log = Path(tmp) / 'gunicorn-access.log'
+            now = time.strftime('%d/%b/%Y:%H:%M:%S %z', time.localtime())
+            log.write_text(''.join([
+                f'127.0.0.1 [{now}] "GET /home/ HTTP/1.1" 200 1200 0.004\n',
+                f'127.0.0.1 [{now}] "POST /api/uploads/ HTTP/1.1" 200 80 0.020\n',
+                f'127.0.0.1 [{now}] "GET /packs/test-pack/ HTTP/1.1" 404 300 0.001\n',
+                f'10.0.0.5 [{now}] "GET /api/stats/ HTTP/1.1" 500 50 0.150\n',
+            ]))
+            with unittest.mock.patch.object(views, 'GUNICORN_ACCESS_LOG', log):
+                stats = views._traffic_stats()
+        self.assertIsNotNone(stats)
+        self.assertEqual(stats['requests_total'], 4)
+        self.assertEqual(stats['requests_min'], 4)
+        self.assertEqual(stats['status'], {'2xx': 2, '3xx': 0, '4xx': 1, '5xx': 1})
+        self.assertEqual(stats['max_ms'], 150.0)
+        self.assertEqual(len(stats['recent']), 4)
+        self.assertEqual(stats['recent'][0]['status'], 500)
+
+    def test_missing_log_returns_none(self):
+        from landing import views
+        with tempfile.TemporaryDirectory() as tmp:
+            log = Path(tmp) / 'nope.log'
+            with unittest.mock.patch.object(views, 'GUNICORN_ACCESS_LOG', log):
+                self.assertIsNone(views._traffic_stats())
@@ -1,9 +1,13 @@
 import hmac
+import os
 import platform
 import random
+import re
 import socket
+import statistics
 import time
 from datetime import timedelta
+from pathlib import Path

 from django import get_version as django_version
 from django.conf import settings
@@ -103,6 +107,30 @@ def _fmt_bytes(n):
        size /= 1024


+def _folder_usage(path):
+    """Sum the allocated bytes of every file under ``path`` (like ``du -s``).
+
+    Uses st_blocks (ext4/Unix allocation) when available, otherwise the
+    apparent file size. Returns None if the path doesn't exist or can't be
+    walked."""
+    root = Path(path)
+    if not root.exists():
+        return None
+    total = 0
+    try:
+        for dirpath, dirnames, filenames in os.walk(root):
+            for name in filenames:
+                try:
+                    st = os.stat(os.path.join(dirpath, name))
+                except OSError:
+                    continue
+                blocks = getattr(st, 'st_blocks', 0)
+                total += blocks * 512 if blocks else st.st_size
+    except OSError:
+        return None
+    return total
+
+
 def _system_snapshot():
    """Collect a one-shot system/performance snapshot via psutil."""
    try:
@@ -137,12 +165,24 @@ def _system_snapshot():
        for label, path in (('Root /', '/'), ('Code', str(settings.BASE_DIR)), ('Media', str(settings.MEDIA_ROOT))):
            try:
                d = psutil.disk_usage(path)
-                snapshot['disks'][label] = {
-                    'path': path, 'total': d.total, 'used': d.used,
-                    'free': d.free, 'percent': d.percent,
-                }
            except OSError:
                snapshot['disks'][label] = None
+                continue
+            # For folders (Code/Media) report the space the folder itself
+            # takes (like `du`), falling back to the whole-drive usage when the
+            # tree can't be measured. Root / always shows the drive.
+            used = d.used
+            scope = 'drive'
+            if label != 'Root /':
+                folder = _folder_usage(path)
+                if folder is not None:
+                    used = folder
+                    scope = 'folder'
+            snapshot['disks'][label] = {
+                'path': path, 'total': d.total, 'used': used,
+                'free': d.free, 'scope': scope,
+                'percent': round(used / d.total * 100, 1) if d.total else 0,
+            }

        try:
            snapshot['net'] = {
@@ -197,6 +237,97 @@ def server_stats(request):

 STATS_CACHE_KEY = 'stats:payload'
 STATS_CACHE_TTL = 20
+GUNICORN_ACCESS_LOG = Path(settings.BASE_DIR).parent / 'logs' / 'gunicorn-access.log'
+
+_ACCESS_LOG_LINE = re.compile(
+    r'^(?P<host>\S+) \[(?P<time>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+)(?: \S+)?" '
+    r'(?P<status>\d{3}) (?P<bytes>\S+) (?P<dur>\d+\.\d+)$'
+)
+
+
+def _traffic_stats():
+    """Parse the tail of gunicorn's access log for live traffic metrics.
+
+    Returns a dict, or None when the log isn't configured/missing (dev server).
+    The gunicorn access-log format is set in run_prod.sh:
+        %(h)s %(t)s "%(r)s" %(s)s %(b)s %(L)s
+    """
+    path = GUNICORN_ACCESS_LOG
+    if not path.exists():
+        return None
+    entries = []
+    try:
+        with open(path, 'rb') as f:
+            size = f.seek(0, 2)
+            chunk = max(0, size - 1_048_576)
+            f.seek(chunk)
+            if chunk:
+                f.readline()  # drop a possibly-split leading line
+            data = f.read().decode('utf-8', 'replace')
+    except OSError:
+        return None
+
+    lines = data.splitlines()[-2000:]
+    for line in lines:
+        m = _ACCESS_LOG_LINE.match(line)
+        if not m:
+            continue
+        try:
+            ts = tz.datetime.strptime(m.group('time'), '%d/%b/%Y:%H:%M:%S %z')
+        except ValueError:
+            continue
+        try:
+            dur_ms = round(float(m.group('dur')) * 1000, 1)
+        except ValueError:
+            dur_ms = None
+        entries.append({
+            'ts': ts,
+            'method': m.group('method'),
+            'path': m.group('path'),
+            'status': int(m.group('status')),
+            'dur_ms': dur_ms,
+        })
+    if not entries:
+        return None
+
+    now = tz.now()
+    cutoff = now - timedelta(minutes=1)
+    recent_minute = [e for e in entries if e['ts'] >= cutoff]
+    durations = [e['dur_ms'] for e in entries if e['dur_ms'] is not None]
+
+    status = {'2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0}
+    for e in entries:
+        code = e['status']
+        if 200 <= code < 300:
+            status['2xx'] += 1
+        elif 300 <= code < 400:
+            status['3xx'] += 1
+        elif 400 <= code < 500:
+            status['4xx'] += 1
+        else:
+            status['5xx'] += 1
+
+    recent = [
+        {
+            'time': e['ts'].isoformat(timespec='seconds'),
+            'method': e['method'],
+            'path': e['path'],
+            'status': e['status'],
+            'dur_ms': e['dur_ms'],
+        }
+        for e in entries[-10:][::-1]
+    ]
+
+    return {
+        'requests_total': len(entries),
+        'requests_min': len(recent_minute),
+        'avg_ms': round(statistics.fmean(durations), 1) if durations else None,
+        'p50_ms': round(statistics.median(durations), 1) if durations else None,
+        'max_ms': round(max(durations), 1) if durations else None,
+        'status': status,
+        'recent': recent,
+    }
+


@login_required
@@ -326,5 +457,6 @@ def _collect_stats(page_gen_ms=None):
        'content': content,
        'categories_list': categories_list,
        'project_categories': project_categories,
+        'traffic': _traffic_stats(),
        'recent': recent,
    }
@@ -55,7 +55,7 @@
        <h2><i class="fas fa-hdd"></i> Disk</h2>
        {% for label in disk_labels %}
        <div data-disk="{{ label }}">
-            <p class="stat-bar-label">{{ label }} <span class="stat-muted" data-f="path">…</span> — <span data-f="percent">…</span>% used</p>
+            <p class="stat-bar-label">{{ label }} <span class="stat-muted" data-f="path">…</span> — <span data-f="percent">…</span>% used <span class="stat-muted" data-f="scope"></span></p>
            <div class="bar-track"><div class="bar-fill" data-f="bar"></div></div>
            <p class="stat-bar-caption"><span data-f="used">…</span> of <span data-f="total">…</span> · <span data-f="free">…</span> free</p>
        </div>
@@ -80,6 +80,13 @@
        </table>
    </section>

+    <section class="card" id="traffic-card">
+        <h2><i class="fas fa-traffic-light"></i> Traffic</h2>
+        <div id="traffic-body">
+            <p class="empty-hint">Loading…</p>
+        </div>
+    </section>
+
    <section class="card">
        <h2><i class="fas fa-database"></i> Site content</h2>
        <div class="stat-tiles">
@@ -206,6 +213,7 @@
            }
            setF('path', disk.path);
            setF('percent', disk.percent);
+            setF('scope', disk.scope === 'folder' ? '· folder size' : '');
            if (bar) bar.style.width = disk.percent + '%';
            setF('used', fmtBytes(disk.used));
            setF('total', fmtBytes(disk.total));
@@ -240,6 +248,43 @@
            ul.innerHTML = items.length ? lists[key](items) : '<li class="empty-hint">None yet.</li>';
        });
    }
+    function buildTraffic(d) {
+        const body = document.getElementById('traffic-body');
+        if (!body) return;
+        const t = d.traffic;
+        if (!t) {
+            body.innerHTML = '<p class="empty-hint">Gunicorn access log is not enabled — start via run_prod.sh to see live traffic.</p>';
+            return;
+        }
+        const st = t.status || {};
+        const dur = (v) => v != null ? v + ' ms' : '—';
+        body.innerHTML =
+            '<div class="stat-tiles">' +
+            `<div class="stat-tile"><span class="stat-tile-value">${t.requests_min}</span><span class="stat-tile-label">Req / min</span></div>` +
+            `<div class="stat-tile"><span class="stat-tile-value">${t.requests_total}</span><span class="stat-tile-label">Requests (tail)</span></div>` +
+            `<div class="stat-tile"><span class="stat-tile-value">${dur(t.avg_ms)}</span><span class="stat-tile-label">Avg resp</span></div>` +
+            `<div class="stat-tile"><span class="stat-tile-value">${dur(t.p50_ms)}</span><span class="stat-tile-label">Median resp</span></div>` +
+            `<div class="stat-tile"><span class="stat-tile-value">${dur(t.max_ms)}</span><span class="stat-tile-label">Max resp</span></div>` +
+            '</div>' +
+            '<p class="stat-line">' +
+            `<span class="stat-chip">2xx: ${st['2xx'] || 0}</span>` +
+            `<span class="stat-chip">3xx: ${st['3xx'] || 0}</span>` +
+            `<span class="stat-chip">4xx: ${st['4xx'] || 0}</span>` +
+            `<span class="stat-chip">5xx: ${st['5xx'] || 0}</span>` +
+            '</p>' +
+            '<h3 class="stat-subhead">Recent requests</h3>' +
+            '<ul class="stat-list">' +
+            (t.recent || []).map(r => {
+                const cls = r.status >= 500 ? 'stat-down' : (r.status >= 400 ? 'stat-muted' : 'stat-up');
+                return `<li><span class="stat-muted">${timeAgo(r.time)}</span> <code class="stat-mono">${esc(r.method)}</code> ` +
+                    `<span class="stat-mono">${esc(r.path)}</span> ` +
+                    `<span class="${cls}">${r.status}</span>` +
+                    (r.dur_ms != null ? ` <span class="stat-muted">${r.dur_ms} ms</span>` : '') +
+                    '</li>';
+            }).join('') +
+            '</ul>';
+    }
+
    function apply(d) {
        const a = d.app || {};
        setHtml('app-env', a.app_env + (a.debug ? ' <span class="badge badge-staff">DEBUG</span>' : ''));
@@ -275,6 +320,7 @@
            buildProcesses(s.top_mem, 'top-mem');
        }
        setTiles(d);
+        buildTraffic(d);
        buildRecent(d);
        set('stats-updated', new Date(d.generated_at).toLocaleTimeString());
    }
@@ -36,8 +36,13 @@ echo "Rebuilding vanilla entity-texture index..."
 # Collect static files (served by gunicorn via whitenoise)
 "$PYTHON_BIN" manage.py collectstatic --noinput

+# Directory for gunicorn's access log (parsed by the stats page).
+mkdir -p "$SCRIPT_DIR/logs"
+
 echo "Starting Gunicorn on 0.0.0.0:8000..."
 exec "$GUNICORN_BIN" \
    --workers 3 \
    --bind 0.0.0.0:8000 \
+    --access-logfile "$SCRIPT_DIR/logs/gunicorn-access.log" \
+    --access-logformat '%(h)s %(t)s "%(r)s" %(s)s %(b)s %(L)s' \
    common.wsgi:application