finished stats page

nonpacks/landing/views.py

@@ -1,9 +1,13 @@
 import hmac
+import os
 import platform
 import random
+import re
 import socket
+import statistics
 import time
 from datetime import timedelta
+from pathlib import Path
 
 from django import get_version as django_version
 from django.conf import settings
@@ -103,6 +107,30 @@ def _fmt_bytes(n):
         size /= 1024
 
 
+def _folder_usage(path):
+    """Sum the allocated bytes of every file under ``path`` (like ``du -s``).
+
+    Uses st_blocks (ext4/Unix allocation) when available, otherwise the
+    apparent file size. Returns None if the path doesn't exist or can't be
+    walked."""
+    root = Path(path)
+    if not root.exists():
+        return None
+    total = 0
+    try:
+        for dirpath, dirnames, filenames in os.walk(root):
+            for name in filenames:
+                try:
+                    st = os.stat(os.path.join(dirpath, name))
+                except OSError:
+                    continue
+                blocks = getattr(st, 'st_blocks', 0)
+                total += blocks * 512 if blocks else st.st_size
+    except OSError:
+        return None
+    return total
+
+
 def _system_snapshot():
     """Collect a one-shot system/performance snapshot via psutil."""
     try:
@@ -137,12 +165,24 @@ def _system_snapshot():
         for label, path in (('Root /', '/'), ('Code', str(settings.BASE_DIR)), ('Media', str(settings.MEDIA_ROOT))):
             try:
                 d = psutil.disk_usage(path)
-                snapshot['disks'][label] = {
-                    'path': path, 'total': d.total, 'used': d.used,
-                    'free': d.free, 'percent': d.percent,
-                }
             except OSError:
                 snapshot['disks'][label] = None
+                continue
+            # For folders (Code/Media) report the space the folder itself
+            # takes (like `du`), falling back to the whole-drive usage when the
+            # tree can't be measured. Root / always shows the drive.
+            used = d.used
+            scope = 'drive'
+            if label != 'Root /':
+                folder = _folder_usage(path)
+                if folder is not None:
+                    used = folder
+                    scope = 'folder'
+            snapshot['disks'][label] = {
+                'path': path, 'total': d.total, 'used': used,
+                'free': d.free, 'scope': scope,
+                'percent': round(used / d.total * 100, 1) if d.total else 0,
+            }
 
         try:
             snapshot['net'] = {
@@ -197,6 +237,97 @@ def server_stats(request):
 
 STATS_CACHE_KEY = 'stats:payload'
 STATS_CACHE_TTL = 20
+GUNICORN_ACCESS_LOG = Path(settings.BASE_DIR).parent / 'logs' / 'gunicorn-access.log'
+
+_ACCESS_LOG_LINE = re.compile(
+    r'^(?P<host>\S+) \[(?P<time>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+)(?: \S+)?" '
+    r'(?P<status>\d{3}) (?P<bytes>\S+) (?P<dur>\d+\.\d+)$'
+)
+
+
+def _traffic_stats():
+    """Parse the tail of gunicorn's access log for live traffic metrics.
+
+    Returns a dict, or None when the log isn't configured/missing (dev server).
+    The gunicorn access-log format is set in run_prod.sh:
+        %(h)s %(t)s "%(r)s" %(s)s %(b)s %(L)s
+    """
+    path = GUNICORN_ACCESS_LOG
+    if not path.exists():
+        return None
+    entries = []
+    try:
+        with open(path, 'rb') as f:
+            size = f.seek(0, 2)
+            chunk = max(0, size - 1_048_576)
+            f.seek(chunk)
+            if chunk:
+                f.readline()  # drop a possibly-split leading line
+            data = f.read().decode('utf-8', 'replace')
+    except OSError:
+        return None
+
+    lines = data.splitlines()[-2000:]
+    for line in lines:
+        m = _ACCESS_LOG_LINE.match(line)
+        if not m:
+            continue
+        try:
+            ts = tz.datetime.strptime(m.group('time'), '%d/%b/%Y:%H:%M:%S %z')
+        except ValueError:
+            continue
+        try:
+            dur_ms = round(float(m.group('dur')) * 1000, 1)
+        except ValueError:
+            dur_ms = None
+        entries.append({
+            'ts': ts,
+            'method': m.group('method'),
+            'path': m.group('path'),
+            'status': int(m.group('status')),
+            'dur_ms': dur_ms,
+        })
+    if not entries:
+        return None
+
+    now = tz.now()
+    cutoff = now - timedelta(minutes=1)
+    recent_minute = [e for e in entries if e['ts'] >= cutoff]
+    durations = [e['dur_ms'] for e in entries if e['dur_ms'] is not None]
+
+    status = {'2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0}
+    for e in entries:
+        code = e['status']
+        if 200 <= code < 300:
+            status['2xx'] += 1
+        elif 300 <= code < 400:
+            status['3xx'] += 1
+        elif 400 <= code < 500:
+            status['4xx'] += 1
+        else:
+            status['5xx'] += 1
+
+    recent = [
+        {
+            'time': e['ts'].isoformat(timespec='seconds'),
+            'method': e['method'],
+            'path': e['path'],
+            'status': e['status'],
+            'dur_ms': e['dur_ms'],
+        }
+        for e in entries[-10:][::-1]
+    ]
+
+    return {
+        'requests_total': len(entries),
+        'requests_min': len(recent_minute),
+        'avg_ms': round(statistics.fmean(durations), 1) if durations else None,
+        'p50_ms': round(statistics.median(durations), 1) if durations else None,
+        'max_ms': round(max(durations), 1) if durations else None,
+        'status': status,
+        'recent': recent,
+    }
+
 
 
 @login_required
@@ -326,5 +457,6 @@ def _collect_stats(page_gen_ms=None):
         'content': content,
         'categories_list': categories_list,
         'project_categories': project_categories,
+        'traffic': _traffic_stats(),
         'recent': recent,
     }