Compare commits
4
Commits
447e5dc4df
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0877f6f5a | ||
|
|
363b0b9998 | ||
|
|
548fdfebaa | ||
|
|
445cb1b50d |
@@ -4,3 +4,7 @@ DISCORD_TOKEN=
|
||||
# Optional tuning
|
||||
CHECK_INTERVAL=60
|
||||
CHECK_TIMEOUT=10
|
||||
|
||||
# Targets file (hot-loaded each scan) and its auto-maintained last-good copy
|
||||
# TARGETS_FILE=targets.json
|
||||
# TARGETS_BACKUP_FILE=targets.json.bak
|
||||
|
||||
@@ -3,3 +3,4 @@ venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.status_message.json
|
||||
targets.json.bak
|
||||
|
||||
@@ -1,30 +1,16 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import aiohttp
|
||||
from mcstatus import JavaServer
|
||||
|
||||
TARGETS = [
|
||||
{
|
||||
'name': 'Gitea',
|
||||
'url': 'https://gitea.rainbow-herring.ts.net/api/healthz',
|
||||
'kind': 'gitea',
|
||||
'critical': True,
|
||||
},
|
||||
{
|
||||
'name': 'Packs Site (Production)',
|
||||
'url': 'https://jakerasp.rainbow-herring.ts.net/healthz/',
|
||||
'kind': 'packs',
|
||||
'critical': True,
|
||||
},
|
||||
{
|
||||
'name': 'Packs Site (Development)',
|
||||
'url': 'https://jakepc.rainbow-herring.ts.net/healthz/',
|
||||
'kind': 'packs',
|
||||
'critical': False,
|
||||
},
|
||||
]
|
||||
import config
|
||||
|
||||
log = logging.getLogger('healthbot')
|
||||
|
||||
GITEA_VERSION_URL = 'https://gitea.rainbow-herring.ts.net/api/v1/version'
|
||||
|
||||
@@ -34,6 +20,75 @@ DOH_RESOLVERS = {
|
||||
}
|
||||
|
||||
|
||||
class TargetsConfigError(Exception):
|
||||
"""Raised when the targets configuration file is missing or invalid."""
|
||||
|
||||
|
||||
def validate_targets(data):
|
||||
"""Validate parsed targets data, returning a normalized list of targets."""
|
||||
if not isinstance(data, list):
|
||||
raise TargetsConfigError('targets must be a JSON list')
|
||||
targets = []
|
||||
for i, entry in enumerate(data):
|
||||
if not isinstance(entry, dict):
|
||||
raise TargetsConfigError(f'entry {i} is not an object')
|
||||
name = entry.get('name')
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise TargetsConfigError(f'entry {i} missing a valid "name"')
|
||||
kind = entry.get('kind')
|
||||
if not isinstance(kind, str) or not kind.strip():
|
||||
raise TargetsConfigError(f'entry {i} missing a valid "kind"')
|
||||
if kind == 'minecraft':
|
||||
host = entry.get('host')
|
||||
port = entry.get('port')
|
||||
if not isinstance(host, str) or not host.strip():
|
||||
raise TargetsConfigError(f'entry {i} missing a valid "host"')
|
||||
if not isinstance(port, int) or not (0 < port < 65536):
|
||||
raise TargetsConfigError(f'entry {i} missing a valid "port"')
|
||||
else:
|
||||
url = entry.get('url')
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
raise TargetsConfigError(f'entry {i} missing a valid "url"')
|
||||
critical = entry.get('critical', True)
|
||||
if not isinstance(critical, bool):
|
||||
raise TargetsConfigError(f'entry {i} "critical" must be true/false')
|
||||
target = {'name': name.strip(), 'kind': kind.strip(), 'critical': critical}
|
||||
target.update({k: entry[k] for k in ('url', 'host', 'port') if k in entry})
|
||||
targets.append(target)
|
||||
return targets
|
||||
|
||||
|
||||
def load_targets():
|
||||
"""Load the live targets file; on success refresh the backup copy.
|
||||
|
||||
Raises TargetsConfigError if the live file is missing or invalid.
|
||||
"""
|
||||
try:
|
||||
with open(config.TARGETS_FILE, 'r') as f:
|
||||
raw = f.read()
|
||||
targets = validate_targets(json.loads(raw))
|
||||
except (OSError, ValueError, TargetsConfigError) as exc:
|
||||
raise TargetsConfigError(
|
||||
f'invalid targets file {config.TARGETS_FILE}: {exc}'
|
||||
) from exc
|
||||
try:
|
||||
shutil.copyfile(config.TARGETS_FILE, config.TARGETS_BACKUP_FILE)
|
||||
except OSError as exc:
|
||||
log.warning('Could not back up targets file: %s', exc)
|
||||
return targets
|
||||
|
||||
|
||||
def load_backup_targets():
|
||||
"""Load the last known-good copy of the targets file."""
|
||||
try:
|
||||
with open(config.TARGETS_BACKUP_FILE, 'r') as f:
|
||||
return validate_targets(json.load(f))
|
||||
except (OSError, ValueError, TargetsConfigError) as exc:
|
||||
raise TargetsConfigError(
|
||||
f'backup targets file {config.TARGETS_BACKUP_FILE} also invalid: {exc}'
|
||||
) from exc
|
||||
|
||||
|
||||
async def _get(session, url, timeout):
|
||||
started = time.monotonic()
|
||||
async with session.get(
|
||||
@@ -78,11 +133,12 @@ async def _resolve_public(session, resolver_url, host, timeout):
|
||||
return {'ok': True, 'ips': [a['data'] for a in answers], 'error': ''}
|
||||
|
||||
|
||||
async def check_target(session, target, timeout):
|
||||
async def check_minecraft(session, target, timeout):
|
||||
host, port = target['host'], target['port']
|
||||
result = {
|
||||
'name': target['name'],
|
||||
'kind': target.get('kind', ''),
|
||||
'url': target['url'],
|
||||
'kind': 'minecraft',
|
||||
'url': f'{host}:{port}',
|
||||
'critical': target.get('critical', True),
|
||||
'up': False,
|
||||
'status_code': None,
|
||||
@@ -91,39 +147,84 @@ async def check_target(session, target, timeout):
|
||||
'version': None,
|
||||
'detail': '',
|
||||
'dns': {},
|
||||
'players': None,
|
||||
'motd': None,
|
||||
}
|
||||
try:
|
||||
status, latency_ms, body = await _get(session, target['url'], timeout)
|
||||
result['status_code'] = status
|
||||
result['latency_ms'] = latency_ms
|
||||
|
||||
if status == 200:
|
||||
data = {}
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
result['up'] = data.get('status') == 'pass'
|
||||
if data.get('db') is not None:
|
||||
result['db'] = data['db']
|
||||
else:
|
||||
result['detail'] = f'HTTP {status}'
|
||||
server = JavaServer(host, port, timeout=timeout)
|
||||
status = await server.async_status()
|
||||
except asyncio.TimeoutError:
|
||||
result['detail'] = 'timeout'
|
||||
except aiohttp.ClientError:
|
||||
return result
|
||||
except OSError:
|
||||
result['detail'] = 'connection error'
|
||||
return result
|
||||
except Exception as exc:
|
||||
result['detail'] = type(exc).__name__
|
||||
return result
|
||||
|
||||
if result['up'] and result['kind'] == 'gitea':
|
||||
result['up'] = True
|
||||
result['latency_ms'] = round(status.latency)
|
||||
result['players'] = f"{status.players.online}/{status.players.max}"
|
||||
if status.version and status.version.name:
|
||||
result['version'] = status.version.name
|
||||
try:
|
||||
result['motd'] = status.motd.to_plain()
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
async def check_target(session, target, timeout):
|
||||
if target.get('kind') == 'minecraft':
|
||||
result = await check_minecraft(session, target, timeout)
|
||||
else:
|
||||
result = {
|
||||
'name': target['name'],
|
||||
'kind': target.get('kind', ''),
|
||||
'url': target['url'],
|
||||
'critical': target.get('critical', True),
|
||||
'up': False,
|
||||
'status_code': None,
|
||||
'latency_ms': None,
|
||||
'db': None,
|
||||
'version': None,
|
||||
'detail': '',
|
||||
'dns': {},
|
||||
}
|
||||
try:
|
||||
status, _, body = await _get(session, GITEA_VERSION_URL, timeout)
|
||||
if status == 200:
|
||||
result['version'] = json.loads(body).get('version')
|
||||
except Exception:
|
||||
pass
|
||||
status, latency_ms, body = await _get(session, target['url'], timeout)
|
||||
result['status_code'] = status
|
||||
result['latency_ms'] = latency_ms
|
||||
|
||||
host = urlparse(target['url']).hostname
|
||||
if status == 200:
|
||||
data = {}
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
pass
|
||||
result['up'] = data.get('status') == 'pass'
|
||||
if data.get('db') is not None:
|
||||
result['db'] = data['db']
|
||||
else:
|
||||
result['detail'] = f'HTTP {status}'
|
||||
except asyncio.TimeoutError:
|
||||
result['detail'] = 'timeout'
|
||||
except aiohttp.ClientError:
|
||||
result['detail'] = 'connection error'
|
||||
except Exception as exc:
|
||||
result['detail'] = type(exc).__name__
|
||||
|
||||
if result['up'] and result['kind'] == 'gitea':
|
||||
try:
|
||||
status, _, body = await _get(session, GITEA_VERSION_URL, timeout)
|
||||
if status == 200:
|
||||
result['version'] = json.loads(body).get('version')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
url = target.get('url') or f'//{target["host"]}'
|
||||
host = urlparse(url).hostname
|
||||
for label, resolver_url in DOH_RESOLVERS.items():
|
||||
result['dns'][label] = await _resolve_public(session, resolver_url, host, timeout)
|
||||
|
||||
@@ -131,7 +232,14 @@ async def check_target(session, target, timeout):
|
||||
|
||||
|
||||
async def run_checks(session, timeout):
|
||||
using_backup = False
|
||||
try:
|
||||
targets = load_targets()
|
||||
except TargetsConfigError as exc:
|
||||
log.exception('Targets config problem: %s', exc)
|
||||
targets = load_backup_targets()
|
||||
using_backup = True
|
||||
results = await asyncio.gather(
|
||||
*(check_target(session, target, timeout) for target in TARGETS)
|
||||
*(check_target(session, target, timeout) for target in targets)
|
||||
)
|
||||
return list(results)
|
||||
return list(results), using_backup
|
||||
|
||||
@@ -10,3 +10,6 @@ CHECK_INTERVAL = int(os.getenv('CHECK_INTERVAL', '60'))
|
||||
CHECK_TIMEOUT = float(os.getenv('CHECK_TIMEOUT', '10'))
|
||||
|
||||
STATE_FILE = os.getenv('STATE_FILE', '.status_message.json')
|
||||
|
||||
TARGETS_FILE = os.getenv('TARGETS_FILE', 'targets.json')
|
||||
TARGETS_BACKUP_FILE = os.getenv('TARGETS_BACKUP_FILE', 'targets.json.bak')
|
||||
|
||||
@@ -12,6 +12,23 @@ import config
|
||||
log = logging.getLogger('healthbot')
|
||||
|
||||
|
||||
def _health_color(results):
|
||||
"""Embed color on a green -> yellow -> red gradient based on the share of
|
||||
critical services that are down (0% down = green, 50% = yellow, 100% = red)."""
|
||||
critical = [r for r in results if r.get('critical', True)]
|
||||
if not critical:
|
||||
return discord.Color.from_rgb(0, 255, 0)
|
||||
down = sum(1 for r in critical if not r['up'])
|
||||
ratio = down / len(critical)
|
||||
if ratio <= 0.5:
|
||||
t = ratio / 0.5
|
||||
red, green, blue = round(255 * t), 255, 0
|
||||
else:
|
||||
t = (ratio - 0.5) / 0.5
|
||||
red, green, blue = 255, round(255 * (1 - t)), 0
|
||||
return discord.Color.from_rgb(red, green, blue)
|
||||
|
||||
|
||||
class HealthBot(discord.Client):
|
||||
def __init__(self):
|
||||
intents = discord.Intents.default()
|
||||
@@ -60,9 +77,8 @@ class HealthBot(discord.Client):
|
||||
session = await self._ensure_session()
|
||||
return await checks.run_checks(session, config.CHECK_TIMEOUT)
|
||||
|
||||
def build_embed(self, results):
|
||||
all_up = all(r['up'] for r in results if r.get('critical', True))
|
||||
color = discord.Color.green() if all_up else discord.Color.red()
|
||||
def build_embed(self, results, using_backup=False):
|
||||
color = _health_color(results)
|
||||
embed = discord.Embed(
|
||||
title='Service Health',
|
||||
color=color,
|
||||
@@ -70,13 +86,21 @@ class HealthBot(discord.Client):
|
||||
)
|
||||
for r in results:
|
||||
emoji = '🟢' if r['up'] else '🔴'
|
||||
code = r['status_code'] if r['status_code'] is not None else '—'
|
||||
latency = f"{r['latency_ms']} ms" if r['latency_ms'] is not None else '—'
|
||||
hc = f'{emoji} `HTTP {code}` · {latency}'
|
||||
if r.get('db') is not None:
|
||||
hc += f' · db {r["db"]}'
|
||||
if r.get('version'):
|
||||
hc += f' · v{r["version"]}'
|
||||
if r.get('kind') == 'minecraft':
|
||||
players = r.get('players') or '—'
|
||||
hc = f'{emoji} `{players} players` · {latency}'
|
||||
if r.get('version'):
|
||||
hc += f' · v{r["version"]}'
|
||||
if r.get('motd'):
|
||||
hc += f' · *{r["motd"]}*'
|
||||
else:
|
||||
code = r['status_code'] if r['status_code'] is not None else '—'
|
||||
hc = f'{emoji} `HTTP {code}` · {latency}'
|
||||
if r.get('db') is not None:
|
||||
hc += f' · db {r["db"]}'
|
||||
if r.get('version'):
|
||||
hc += f' · v{r["version"]}'
|
||||
if r.get('detail'):
|
||||
hc += f' · *{r["detail"]}*'
|
||||
|
||||
@@ -91,14 +115,20 @@ class HealthBot(discord.Client):
|
||||
lines.append(f'**DNS resolution ({label})** {val}')
|
||||
|
||||
embed.add_field(name=r['name'], value='\n'.join(lines), inline=False)
|
||||
embed.set_footer(text=f'Updates every {config.CHECK_INTERVAL}s')
|
||||
embed.set_footer(text=self._footer_text(using_backup))
|
||||
return embed
|
||||
|
||||
def _footer_text(self, using_backup):
|
||||
text = f'Updates every {config.CHECK_INTERVAL}s'
|
||||
if using_backup:
|
||||
text += ' · ⚠ targets.json invalid — using last known-good config'
|
||||
return text
|
||||
|
||||
async def update_embeds(self):
|
||||
if not self.status_messages:
|
||||
return
|
||||
results = await self.run_checks()
|
||||
embed = self.build_embed(results)
|
||||
results, using_backup = await self.run_checks()
|
||||
embed = self.build_embed(results, using_backup)
|
||||
|
||||
kept = []
|
||||
for entry in self.status_messages:
|
||||
@@ -148,8 +178,14 @@ bot = HealthBot()
|
||||
@bot.tree.command(name='status', description='Post a live-updating health embed for the services')
|
||||
async def status(interaction: discord.Interaction):
|
||||
await interaction.response.defer()
|
||||
results = await bot.run_checks()
|
||||
embed = bot.build_embed(results)
|
||||
try:
|
||||
results, using_backup = await bot.run_checks()
|
||||
except checks.TargetsConfigError as exc:
|
||||
await interaction.followup.send(
|
||||
f'Could not load targets config: {exc}', ephemeral=True,
|
||||
)
|
||||
return
|
||||
embed = bot.build_embed(results, using_backup)
|
||||
|
||||
existing = next(
|
||||
(m for m in bot.status_messages if m['channel_id'] == interaction.channel_id),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
discord.py
|
||||
aiohttp
|
||||
python-dotenv
|
||||
mcstatus==14.1.0
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
[
|
||||
{
|
||||
"name": "Gitea",
|
||||
"url": "https://gitea.rainbow-herring.ts.net/api/healthz",
|
||||
"kind": "gitea",
|
||||
"critical": true
|
||||
},
|
||||
{
|
||||
"name": "Packs Site (Production)",
|
||||
"url": "https://jakerasp.rainbow-herring.ts.net/healthz/",
|
||||
"kind": "packs",
|
||||
"critical": true
|
||||
},
|
||||
{
|
||||
"name": "Jake's Minecraft Server",
|
||||
"host": "jakelaptop.rainbow-herring.ts.net",
|
||||
"port": 10000,
|
||||
"kind": "minecraft",
|
||||
"critical": true
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user