462 lines
15 KiB
Python
462 lines
15 KiB
Python
import hmac
|
|
import os
|
|
import platform
|
|
import random
|
|
import re
|
|
import socket
|
|
import statistics
|
|
import time
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
|
|
from django import get_version as django_version
|
|
from django.conf import settings
|
|
from django.contrib.auth import get_user_model
|
|
from django.contrib.auth.decorators import login_required
|
|
from django.contrib.sessions.models import Session
|
|
from django.core.cache import cache
|
|
from django.db.models import Count, Sum
|
|
from django.http import HttpResponseForbidden, JsonResponse
|
|
from django.shortcuts import redirect, render
|
|
from django.utils import timezone as tz
|
|
|
|
from common.git_utils import get_git_commit_hash
|
|
from common.middleware import gate_whitelist
|
|
from library.models import (
|
|
Comment,
|
|
FileIndex,
|
|
Project,
|
|
ProjectAsset,
|
|
ProjectDraft,
|
|
Rating,
|
|
Tag,
|
|
TempUpload,
|
|
Version,
|
|
VersionFile,
|
|
)
|
|
|
|
|
|
def _under_lockout(request):
|
|
lock_until = request.session.get('lock_until')
|
|
if lock_until and time.time() < lock_until:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _start_lockout(request):
|
|
duration = random.randint(30, 300)
|
|
request.session['lock_until'] = time.time() + duration
|
|
|
|
|
|
@gate_whitelist
|
|
def gate(request):
|
|
if request.method == 'POST':
|
|
if _under_lockout(request):
|
|
return render(request, 'landing/gate.html', {
|
|
'error': 'Incorrect password.',
|
|
})
|
|
|
|
submitted = request.POST.get('password', '')
|
|
master = settings.PASSWORD or ''
|
|
if hmac.compare_digest(submitted, master):
|
|
request.session['authorized'] = True
|
|
request.session.pop('lock_until', None)
|
|
return redirect('landing:home')
|
|
else:
|
|
_start_lockout(request)
|
|
return render(request, 'landing/gate.html', {
|
|
'error': 'Incorrect password.',
|
|
})
|
|
|
|
if request.session.get('authorized'):
|
|
return redirect('landing:home')
|
|
|
|
return render(request, 'landing/gate.html')
|
|
|
|
|
|
def home(request):
|
|
latest_projects = (
|
|
Project.objects.select_related('owner', 'thumbnail')
|
|
.prefetch_related('tag_links__tag__category', 'versions')
|
|
.order_by('-created_at')[:4]
|
|
)
|
|
total_downloads = Version.objects.aggregate(total=Sum('downloads'))['total'] or 0
|
|
creators = (
|
|
get_user_model().objects.filter(projects__isnull=False).distinct().count()
|
|
)
|
|
context = {
|
|
'latest_projects': latest_projects,
|
|
'stats': {
|
|
'creators': creators,
|
|
'packs': Project.objects.count(),
|
|
'downloads': total_downloads,
|
|
'versions': Version.objects.count(),
|
|
},
|
|
}
|
|
return render(request, 'landing/home.html', context)
|
|
|
|
|
|
def _fmt_bytes(n):
|
|
"""Human-readable byte count."""
|
|
if n is None:
|
|
return '—'
|
|
size = float(n)
|
|
for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
|
|
if size < 1024 or unit == 'TB':
|
|
return f'{size:.1f} {unit}'
|
|
size /= 1024
|
|
|
|
|
|
def _folder_usage(path):
|
|
"""Sum the allocated bytes of every file under ``path`` (like ``du -s``).
|
|
|
|
Uses st_blocks (ext4/Unix allocation) when available, otherwise the
|
|
apparent file size. Returns None if the path doesn't exist or can't be
|
|
walked."""
|
|
root = Path(path)
|
|
if not root.exists():
|
|
return None
|
|
total = 0
|
|
try:
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
for name in filenames:
|
|
try:
|
|
st = os.stat(os.path.join(dirpath, name))
|
|
except OSError:
|
|
continue
|
|
blocks = getattr(st, 'st_blocks', 0)
|
|
total += blocks * 512 if blocks else st.st_size
|
|
except OSError:
|
|
return None
|
|
return total
|
|
|
|
|
|
def _system_snapshot():
|
|
"""Collect a one-shot system/performance snapshot via psutil."""
|
|
try:
|
|
import psutil
|
|
except ImportError:
|
|
return None
|
|
snapshot = {}
|
|
try:
|
|
boot_ts = psutil.boot_time()
|
|
snapshot['boot_time'] = tz.datetime.fromtimestamp(
|
|
boot_ts, tz.get_current_timezone()
|
|
).isoformat(timespec='seconds')
|
|
snapshot['uptime'] = round(time.time() - boot_ts)
|
|
try:
|
|
snapshot['load_avg'] = [round(x, 2) for x in psutil.getloadavg()]
|
|
except Exception:
|
|
snapshot['load_avg'] = None
|
|
snapshot['cpus'] = {
|
|
'logical': psutil.cpu_count(logical=True) or 0,
|
|
'physical': psutil.cpu_count(logical=False) or 0,
|
|
'percent': psutil.cpu_percent(interval=0.1),
|
|
}
|
|
vm = psutil.virtual_memory()
|
|
snapshot['memory'] = {
|
|
'total': vm.total, 'used': vm.used,
|
|
'available': vm.available, 'percent': vm.percent,
|
|
}
|
|
sw = psutil.swap_memory()
|
|
snapshot['swap'] = {'total': sw.total, 'used': sw.used, 'percent': sw.percent}
|
|
|
|
snapshot['disks'] = {}
|
|
for label, path in (('Root /', '/'), ('Code', str(settings.BASE_DIR)), ('Media', str(settings.MEDIA_ROOT))):
|
|
try:
|
|
d = psutil.disk_usage(path)
|
|
except OSError:
|
|
snapshot['disks'][label] = None
|
|
continue
|
|
# For folders (Code/Media) report the space the folder itself
|
|
# takes (like `du`), falling back to the whole-drive usage when the
|
|
# tree can't be measured. Root / always shows the drive.
|
|
used = d.used
|
|
scope = 'drive'
|
|
if label != 'Root /':
|
|
folder = _folder_usage(path)
|
|
if folder is not None:
|
|
used = folder
|
|
scope = 'folder'
|
|
snapshot['disks'][label] = {
|
|
'path': path, 'total': d.total, 'used': used,
|
|
'free': d.free, 'scope': scope,
|
|
'percent': round(used / d.total * 100, 1) if d.total else 0,
|
|
}
|
|
|
|
try:
|
|
snapshot['net'] = {
|
|
'sent': psutil.net_io_counters().bytes_sent,
|
|
'recv': psutil.net_io_counters().bytes_recv,
|
|
}
|
|
except Exception:
|
|
snapshot['net'] = None
|
|
|
|
running = 0
|
|
top_cpu = []
|
|
top_mem = []
|
|
gunicorn = 0
|
|
for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'cpu_percent', 'memory_percent']):
|
|
try:
|
|
running += 1
|
|
info = proc.info
|
|
cmd = ' '.join(info.get('cmdline') or [])
|
|
if 'gunicorn' in cmd:
|
|
gunicorn += 1
|
|
top_cpu.append({
|
|
'name': info.get('name') or '?', 'pid': info.get('pid'),
|
|
'value': round(info.get('cpu_percent') or 0, 1),
|
|
})
|
|
top_mem.append({
|
|
'name': info.get('name') or '?', 'pid': info.get('pid'),
|
|
'value': round(info.get('memory_percent') or 0, 1),
|
|
})
|
|
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
|
continue
|
|
snapshot['processes'] = running
|
|
snapshot['gunicorn'] = gunicorn
|
|
top_cpu.sort(key=lambda x: x['value'], reverse=True)
|
|
top_mem.sort(key=lambda x: x['value'], reverse=True)
|
|
snapshot['top_cpu'] = top_cpu[:6]
|
|
snapshot['top_mem'] = top_mem[:6]
|
|
except Exception:
|
|
return None
|
|
return snapshot
|
|
|
|
|
|
@login_required
|
|
def server_stats(request):
|
|
"""Staff-only stats page. Renders a lightweight shell; the snapshot data
|
|
is fetched from the JSON endpoint so page loads stay cheap."""
|
|
if not request.user.is_staff:
|
|
return HttpResponseForbidden('Only staff can view server stats.')
|
|
return render(request, 'landing/server_stats.html', {
|
|
'disk_labels': ['Root /', 'Code', 'Media'],
|
|
})
|
|
|
|
|
|
STATS_CACHE_KEY = 'stats:payload'
|
|
STATS_CACHE_TTL = 20
|
|
GUNICORN_ACCESS_LOG = Path(settings.BASE_DIR).parent / 'logs' / 'gunicorn-access.log'
|
|
|
|
_ACCESS_LOG_LINE = re.compile(
|
|
r'^(?P<host>\S+) \[(?P<time>[^\]]+)\] "(?P<method>\S+) (?P<path>\S+)(?: \S+)?" '
|
|
r'(?P<status>\d{3}) (?P<bytes>\S+) (?P<dur>\d+\.\d+)$'
|
|
)
|
|
|
|
|
|
def _traffic_stats():
|
|
"""Parse the tail of gunicorn's access log for live traffic metrics.
|
|
|
|
Returns a dict, or None when the log isn't configured/missing (dev server).
|
|
The gunicorn access-log format is set in run_prod.sh:
|
|
%(h)s %(t)s "%(r)s" %(s)s %(b)s %(L)s
|
|
"""
|
|
path = GUNICORN_ACCESS_LOG
|
|
if not path.exists():
|
|
return None
|
|
entries = []
|
|
try:
|
|
with open(path, 'rb') as f:
|
|
size = f.seek(0, 2)
|
|
chunk = max(0, size - 1_048_576)
|
|
f.seek(chunk)
|
|
if chunk:
|
|
f.readline() # drop a possibly-split leading line
|
|
data = f.read().decode('utf-8', 'replace')
|
|
except OSError:
|
|
return None
|
|
|
|
lines = data.splitlines()[-2000:]
|
|
for line in lines:
|
|
m = _ACCESS_LOG_LINE.match(line)
|
|
if not m:
|
|
continue
|
|
try:
|
|
ts = tz.datetime.strptime(m.group('time'), '%d/%b/%Y:%H:%M:%S %z')
|
|
except ValueError:
|
|
continue
|
|
try:
|
|
dur_ms = round(float(m.group('dur')) * 1000, 1)
|
|
except ValueError:
|
|
dur_ms = None
|
|
entries.append({
|
|
'ts': ts,
|
|
'method': m.group('method'),
|
|
'path': m.group('path'),
|
|
'status': int(m.group('status')),
|
|
'dur_ms': dur_ms,
|
|
})
|
|
if not entries:
|
|
return None
|
|
|
|
now = tz.now()
|
|
cutoff = now - timedelta(minutes=1)
|
|
recent_minute = [e for e in entries if e['ts'] >= cutoff]
|
|
durations = [e['dur_ms'] for e in entries if e['dur_ms'] is not None]
|
|
|
|
status = {'2xx': 0, '3xx': 0, '4xx': 0, '5xx': 0}
|
|
for e in entries:
|
|
code = e['status']
|
|
if 200 <= code < 300:
|
|
status['2xx'] += 1
|
|
elif 300 <= code < 400:
|
|
status['3xx'] += 1
|
|
elif 400 <= code < 500:
|
|
status['4xx'] += 1
|
|
else:
|
|
status['5xx'] += 1
|
|
|
|
recent = [
|
|
{
|
|
'time': e['ts'].isoformat(timespec='seconds'),
|
|
'method': e['method'],
|
|
'path': e['path'],
|
|
'status': e['status'],
|
|
'dur_ms': e['dur_ms'],
|
|
}
|
|
for e in entries[-10:][::-1]
|
|
]
|
|
|
|
return {
|
|
'requests_total': len(entries),
|
|
'requests_min': len(recent_minute),
|
|
'avg_ms': round(statistics.fmean(durations), 1) if durations else None,
|
|
'p50_ms': round(statistics.median(durations), 1) if durations else None,
|
|
'max_ms': round(max(durations), 1) if durations else None,
|
|
'status': status,
|
|
'recent': recent,
|
|
}
|
|
|
|
|
|
|
|
@login_required
|
|
def server_stats_api(request):
|
|
"""JSON snapshot of server state / performance / recent activity.
|
|
|
|
The payload is cached server-side for STATS_CACHE_TTL seconds so polling
|
|
clients share one computation per interval instead of hammering the DB."""
|
|
if not request.user.is_staff:
|
|
return JsonResponse({'detail': 'Only staff can view server stats.'}, status=403)
|
|
payload = cache.get(STATS_CACHE_KEY)
|
|
if payload is None:
|
|
start = time.time()
|
|
payload = _collect_stats()
|
|
payload['app']['page_gen_ms'] = round((time.time() - start) * 1000, 1)
|
|
cache.set(STATS_CACHE_KEY, payload, STATS_CACHE_TTL)
|
|
return JsonResponse(payload)
|
|
|
|
|
|
def _collect_stats(page_gen_ms=None):
|
|
"""One-shot, JSON-safe snapshot of server state and recent activity."""
|
|
now = tz.now()
|
|
users = get_user_model().objects
|
|
|
|
category_breakdown = dict(
|
|
Project.objects.values_list('category').annotate(n=Count('pk')).order_by()
|
|
)
|
|
project_categories = [c for c, _ in Project.CATEGORY_CHOICES]
|
|
categories_list = [
|
|
[c, category_breakdown.get(c, 0)] for c in project_categories
|
|
]
|
|
|
|
try:
|
|
import importlib.metadata as _md
|
|
gunicorn_version = _md.version('gunicorn')
|
|
except Exception:
|
|
gunicorn_version = 'unknown'
|
|
|
|
content = {
|
|
'users': users.count(),
|
|
'staff': users.filter(is_staff=True).count(),
|
|
'superusers': users.filter(is_superuser=True).count(),
|
|
'active_24h': users.filter(last_login__gte=now - timedelta(hours=24)).count(),
|
|
'active_7d': users.filter(last_login__gte=now - timedelta(days=7)).count(),
|
|
'projects': sum(category_breakdown.get(c, 0) for c in project_categories),
|
|
'categories': category_breakdown,
|
|
'versions': Version.objects.count(),
|
|
'files': FileIndex.objects.count(),
|
|
'storage': _fmt_bytes(
|
|
FileIndex.objects.aggregate(s=Sum('size'))['s'] or 0
|
|
),
|
|
'assets': ProjectAsset.objects.count(),
|
|
'downloads': Version.objects.aggregate(t=Sum('downloads'))['t'] or 0,
|
|
'ratings': Rating.objects.count(),
|
|
'ratings_positive': Rating.objects.filter(value=True).count(),
|
|
'ratings_negative': Rating.objects.filter(value=False).count(),
|
|
'comments': Comment.objects.count(),
|
|
'tags': Tag.objects.count(),
|
|
'sessions': Session.objects.filter(expire_date__gte=now).count(),
|
|
'pending_uploads': TempUpload.objects.filter(status='pending').count(),
|
|
'drafts': ProjectDraft.objects.count(),
|
|
}
|
|
|
|
def _iso(dt):
|
|
return dt.isoformat(timespec='seconds')
|
|
|
|
recent = {
|
|
'versions': [
|
|
{
|
|
'project': v.project.title,
|
|
'slug': v.project.slug,
|
|
'version': v.version_name,
|
|
'created_at': _iso(v.created_at),
|
|
}
|
|
for v in Version.objects.select_related('project').order_by('-created_at')[:8]
|
|
],
|
|
'comments': [
|
|
{
|
|
'user': c.user.username,
|
|
'project': c.project.title,
|
|
'slug': c.project.slug,
|
|
'body': c.body,
|
|
'created_at': _iso(c.created_at),
|
|
}
|
|
for c in Comment.objects.select_related('user', 'project').order_by('-created_at')[:8]
|
|
],
|
|
'projects': [
|
|
{
|
|
'title': p.title,
|
|
'slug': p.slug,
|
|
'category': p.category,
|
|
'owner': p.owner.username,
|
|
'created_at': _iso(p.created_at),
|
|
}
|
|
for p in Project.objects.select_related('owner').order_by('-created_at')[:5]
|
|
],
|
|
'ratings': [
|
|
{
|
|
'user': r.user.username,
|
|
'project': r.project.title,
|
|
'slug': r.project.slug,
|
|
'value': r.value,
|
|
'created_at': _iso(r.created_at),
|
|
}
|
|
for r in Rating.objects.select_related('user', 'project').order_by('-created_at')[:5]
|
|
],
|
|
}
|
|
|
|
app = {
|
|
'django': django_version(),
|
|
'python': platform.python_version(),
|
|
'hostname': socket.gethostname(),
|
|
'app_env': 'dev' if settings.DEBUG else 'prod',
|
|
'debug': settings.DEBUG,
|
|
'git': get_git_commit_hash(),
|
|
'db': settings.DATABASES['default']['ENGINE'].rsplit('.', 1)[-1],
|
|
'tz': settings.TIME_ZONE,
|
|
'media_root': str(settings.MEDIA_ROOT),
|
|
'gunicorn': gunicorn_version,
|
|
'page_gen_ms': page_gen_ms,
|
|
}
|
|
|
|
return {
|
|
'generated_at': _iso(now),
|
|
'system': _system_snapshot(),
|
|
'app': app,
|
|
'content': content,
|
|
'categories_list': categories_list,
|
|
'project_categories': project_categories,
|
|
'traffic': _traffic_stats(),
|
|
'recent': recent,
|
|
} |