Compare commits
4
Commits
447e5dc4df
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0877f6f5a | ||
|
|
363b0b9998 | ||
|
|
548fdfebaa | ||
|
|
445cb1b50d |
@@ -4,3 +4,7 @@ DISCORD_TOKEN=
|
|||||||
# Optional tuning
|
# Optional tuning
|
||||||
CHECK_INTERVAL=60
|
CHECK_INTERVAL=60
|
||||||
CHECK_TIMEOUT=10
|
CHECK_TIMEOUT=10
|
||||||
|
|
||||||
|
# Targets file (hot-loaded each scan) and its auto-maintained last-good copy
|
||||||
|
# TARGETS_FILE=targets.json
|
||||||
|
# TARGETS_BACKUP_FILE=targets.json.bak
|
||||||
|
|||||||
@@ -3,3 +3,4 @@ venv/
|
|||||||
__pycache__/
|
__pycache__/
|
||||||
*.pyc
|
*.pyc
|
||||||
.status_message.json
|
.status_message.json
|
||||||
|
targets.json.bak
|
||||||
|
|||||||
@@ -1,30 +1,16 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
|
import shutil
|
||||||
import time
|
import time
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
from mcstatus import JavaServer
|
||||||
|
|
||||||
TARGETS = [
|
import config
|
||||||
{
|
|
||||||
'name': 'Gitea',
|
log = logging.getLogger('healthbot')
|
||||||
'url': 'https://gitea.rainbow-herring.ts.net/api/healthz',
|
|
||||||
'kind': 'gitea',
|
|
||||||
'critical': True,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'name': 'Packs Site (Production)',
|
|
||||||
'url': 'https://jakerasp.rainbow-herring.ts.net/healthz/',
|
|
||||||
'kind': 'packs',
|
|
||||||
'critical': True,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
'name': 'Packs Site (Development)',
|
|
||||||
'url': 'https://jakepc.rainbow-herring.ts.net/healthz/',
|
|
||||||
'kind': 'packs',
|
|
||||||
'critical': False,
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
GITEA_VERSION_URL = 'https://gitea.rainbow-herring.ts.net/api/v1/version'
|
GITEA_VERSION_URL = 'https://gitea.rainbow-herring.ts.net/api/v1/version'
|
||||||
|
|
||||||
@@ -34,6 +20,75 @@ DOH_RESOLVERS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TargetsConfigError(Exception):
|
||||||
|
"""Raised when the targets configuration file is missing or invalid."""
|
||||||
|
|
||||||
|
|
||||||
|
def validate_targets(data):
|
||||||
|
"""Validate parsed targets data, returning a normalized list of targets."""
|
||||||
|
if not isinstance(data, list):
|
||||||
|
raise TargetsConfigError('targets must be a JSON list')
|
||||||
|
targets = []
|
||||||
|
for i, entry in enumerate(data):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
raise TargetsConfigError(f'entry {i} is not an object')
|
||||||
|
name = entry.get('name')
|
||||||
|
if not isinstance(name, str) or not name.strip():
|
||||||
|
raise TargetsConfigError(f'entry {i} missing a valid "name"')
|
||||||
|
kind = entry.get('kind')
|
||||||
|
if not isinstance(kind, str) or not kind.strip():
|
||||||
|
raise TargetsConfigError(f'entry {i} missing a valid "kind"')
|
||||||
|
if kind == 'minecraft':
|
||||||
|
host = entry.get('host')
|
||||||
|
port = entry.get('port')
|
||||||
|
if not isinstance(host, str) or not host.strip():
|
||||||
|
raise TargetsConfigError(f'entry {i} missing a valid "host"')
|
||||||
|
if not isinstance(port, int) or not (0 < port < 65536):
|
||||||
|
raise TargetsConfigError(f'entry {i} missing a valid "port"')
|
||||||
|
else:
|
||||||
|
url = entry.get('url')
|
||||||
|
if not isinstance(url, str) or not url.strip():
|
||||||
|
raise TargetsConfigError(f'entry {i} missing a valid "url"')
|
||||||
|
critical = entry.get('critical', True)
|
||||||
|
if not isinstance(critical, bool):
|
||||||
|
raise TargetsConfigError(f'entry {i} "critical" must be true/false')
|
||||||
|
target = {'name': name.strip(), 'kind': kind.strip(), 'critical': critical}
|
||||||
|
target.update({k: entry[k] for k in ('url', 'host', 'port') if k in entry})
|
||||||
|
targets.append(target)
|
||||||
|
return targets
|
||||||
|
|
||||||
|
|
||||||
|
def load_targets():
|
||||||
|
"""Load the live targets file; on success refresh the backup copy.
|
||||||
|
|
||||||
|
Raises TargetsConfigError if the live file is missing or invalid.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
with open(config.TARGETS_FILE, 'r') as f:
|
||||||
|
raw = f.read()
|
||||||
|
targets = validate_targets(json.loads(raw))
|
||||||
|
except (OSError, ValueError, TargetsConfigError) as exc:
|
||||||
|
raise TargetsConfigError(
|
||||||
|
f'invalid targets file {config.TARGETS_FILE}: {exc}'
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
shutil.copyfile(config.TARGETS_FILE, config.TARGETS_BACKUP_FILE)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning('Could not back up targets file: %s', exc)
|
||||||
|
return targets
|
||||||
|
|
||||||
|
|
||||||
|
def load_backup_targets():
|
||||||
|
"""Load the last known-good copy of the targets file."""
|
||||||
|
try:
|
||||||
|
with open(config.TARGETS_BACKUP_FILE, 'r') as f:
|
||||||
|
return validate_targets(json.load(f))
|
||||||
|
except (OSError, ValueError, TargetsConfigError) as exc:
|
||||||
|
raise TargetsConfigError(
|
||||||
|
f'backup targets file {config.TARGETS_BACKUP_FILE} also invalid: {exc}'
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
async def _get(session, url, timeout):
|
async def _get(session, url, timeout):
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
async with session.get(
|
async with session.get(
|
||||||
@@ -78,7 +133,52 @@ async def _resolve_public(session, resolver_url, host, timeout):
|
|||||||
return {'ok': True, 'ips': [a['data'] for a in answers], 'error': ''}
|
return {'ok': True, 'ips': [a['data'] for a in answers], 'error': ''}
|
||||||
|
|
||||||
|
|
||||||
|
async def check_minecraft(session, target, timeout):
|
||||||
|
host, port = target['host'], target['port']
|
||||||
|
result = {
|
||||||
|
'name': target['name'],
|
||||||
|
'kind': 'minecraft',
|
||||||
|
'url': f'{host}:{port}',
|
||||||
|
'critical': target.get('critical', True),
|
||||||
|
'up': False,
|
||||||
|
'status_code': None,
|
||||||
|
'latency_ms': None,
|
||||||
|
'db': None,
|
||||||
|
'version': None,
|
||||||
|
'detail': '',
|
||||||
|
'dns': {},
|
||||||
|
'players': None,
|
||||||
|
'motd': None,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
server = JavaServer(host, port, timeout=timeout)
|
||||||
|
status = await server.async_status()
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
result['detail'] = 'timeout'
|
||||||
|
return result
|
||||||
|
except OSError:
|
||||||
|
result['detail'] = 'connection error'
|
||||||
|
return result
|
||||||
|
except Exception as exc:
|
||||||
|
result['detail'] = type(exc).__name__
|
||||||
|
return result
|
||||||
|
|
||||||
|
result['up'] = True
|
||||||
|
result['latency_ms'] = round(status.latency)
|
||||||
|
result['players'] = f"{status.players.online}/{status.players.max}"
|
||||||
|
if status.version and status.version.name:
|
||||||
|
result['version'] = status.version.name
|
||||||
|
try:
|
||||||
|
result['motd'] = status.motd.to_plain()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
async def check_target(session, target, timeout):
|
async def check_target(session, target, timeout):
|
||||||
|
if target.get('kind') == 'minecraft':
|
||||||
|
result = await check_minecraft(session, target, timeout)
|
||||||
|
else:
|
||||||
result = {
|
result = {
|
||||||
'name': target['name'],
|
'name': target['name'],
|
||||||
'kind': target.get('kind', ''),
|
'kind': target.get('kind', ''),
|
||||||
@@ -123,7 +223,8 @@ async def check_target(session, target, timeout):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
host = urlparse(target['url']).hostname
|
url = target.get('url') or f'//{target["host"]}'
|
||||||
|
host = urlparse(url).hostname
|
||||||
for label, resolver_url in DOH_RESOLVERS.items():
|
for label, resolver_url in DOH_RESOLVERS.items():
|
||||||
result['dns'][label] = await _resolve_public(session, resolver_url, host, timeout)
|
result['dns'][label] = await _resolve_public(session, resolver_url, host, timeout)
|
||||||
|
|
||||||
@@ -131,7 +232,14 @@ async def check_target(session, target, timeout):
|
|||||||
|
|
||||||
|
|
||||||
async def run_checks(session, timeout):
|
async def run_checks(session, timeout):
|
||||||
|
using_backup = False
|
||||||
|
try:
|
||||||
|
targets = load_targets()
|
||||||
|
except TargetsConfigError as exc:
|
||||||
|
log.exception('Targets config problem: %s', exc)
|
||||||
|
targets = load_backup_targets()
|
||||||
|
using_backup = True
|
||||||
results = await asyncio.gather(
|
results = await asyncio.gather(
|
||||||
*(check_target(session, target, timeout) for target in TARGETS)
|
*(check_target(session, target, timeout) for target in targets)
|
||||||
)
|
)
|
||||||
return list(results)
|
return list(results), using_backup
|
||||||
|
|||||||
@@ -10,3 +10,6 @@ CHECK_INTERVAL = int(os.getenv('CHECK_INTERVAL', '60'))
|
|||||||
CHECK_TIMEOUT = float(os.getenv('CHECK_TIMEOUT', '10'))
|
CHECK_TIMEOUT = float(os.getenv('CHECK_TIMEOUT', '10'))
|
||||||
|
|
||||||
STATE_FILE = os.getenv('STATE_FILE', '.status_message.json')
|
STATE_FILE = os.getenv('STATE_FILE', '.status_message.json')
|
||||||
|
|
||||||
|
TARGETS_FILE = os.getenv('TARGETS_FILE', 'targets.json')
|
||||||
|
TARGETS_BACKUP_FILE = os.getenv('TARGETS_BACKUP_FILE', 'targets.json.bak')
|
||||||
|
|||||||
@@ -12,6 +12,23 @@ import config
|
|||||||
log = logging.getLogger('healthbot')
|
log = logging.getLogger('healthbot')
|
||||||
|
|
||||||
|
|
||||||
|
def _health_color(results):
|
||||||
|
"""Embed color on a green -> yellow -> red gradient based on the share of
|
||||||
|
critical services that are down (0% down = green, 50% = yellow, 100% = red)."""
|
||||||
|
critical = [r for r in results if r.get('critical', True)]
|
||||||
|
if not critical:
|
||||||
|
return discord.Color.from_rgb(0, 255, 0)
|
||||||
|
down = sum(1 for r in critical if not r['up'])
|
||||||
|
ratio = down / len(critical)
|
||||||
|
if ratio <= 0.5:
|
||||||
|
t = ratio / 0.5
|
||||||
|
red, green, blue = round(255 * t), 255, 0
|
||||||
|
else:
|
||||||
|
t = (ratio - 0.5) / 0.5
|
||||||
|
red, green, blue = 255, round(255 * (1 - t)), 0
|
||||||
|
return discord.Color.from_rgb(red, green, blue)
|
||||||
|
|
||||||
|
|
||||||
class HealthBot(discord.Client):
|
class HealthBot(discord.Client):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
intents = discord.Intents.default()
|
intents = discord.Intents.default()
|
||||||
@@ -60,9 +77,8 @@ class HealthBot(discord.Client):
|
|||||||
session = await self._ensure_session()
|
session = await self._ensure_session()
|
||||||
return await checks.run_checks(session, config.CHECK_TIMEOUT)
|
return await checks.run_checks(session, config.CHECK_TIMEOUT)
|
||||||
|
|
||||||
def build_embed(self, results):
|
def build_embed(self, results, using_backup=False):
|
||||||
all_up = all(r['up'] for r in results if r.get('critical', True))
|
color = _health_color(results)
|
||||||
color = discord.Color.green() if all_up else discord.Color.red()
|
|
||||||
embed = discord.Embed(
|
embed = discord.Embed(
|
||||||
title='Service Health',
|
title='Service Health',
|
||||||
color=color,
|
color=color,
|
||||||
@@ -70,8 +86,16 @@ class HealthBot(discord.Client):
|
|||||||
)
|
)
|
||||||
for r in results:
|
for r in results:
|
||||||
emoji = '🟢' if r['up'] else '🔴'
|
emoji = '🟢' if r['up'] else '🔴'
|
||||||
code = r['status_code'] if r['status_code'] is not None else '—'
|
|
||||||
latency = f"{r['latency_ms']} ms" if r['latency_ms'] is not None else '—'
|
latency = f"{r['latency_ms']} ms" if r['latency_ms'] is not None else '—'
|
||||||
|
if r.get('kind') == 'minecraft':
|
||||||
|
players = r.get('players') or '—'
|
||||||
|
hc = f'{emoji} `{players} players` · {latency}'
|
||||||
|
if r.get('version'):
|
||||||
|
hc += f' · v{r["version"]}'
|
||||||
|
if r.get('motd'):
|
||||||
|
hc += f' · *{r["motd"]}*'
|
||||||
|
else:
|
||||||
|
code = r['status_code'] if r['status_code'] is not None else '—'
|
||||||
hc = f'{emoji} `HTTP {code}` · {latency}'
|
hc = f'{emoji} `HTTP {code}` · {latency}'
|
||||||
if r.get('db') is not None:
|
if r.get('db') is not None:
|
||||||
hc += f' · db {r["db"]}'
|
hc += f' · db {r["db"]}'
|
||||||
@@ -91,14 +115,20 @@ class HealthBot(discord.Client):
|
|||||||
lines.append(f'**DNS resolution ({label})** {val}')
|
lines.append(f'**DNS resolution ({label})** {val}')
|
||||||
|
|
||||||
embed.add_field(name=r['name'], value='\n'.join(lines), inline=False)
|
embed.add_field(name=r['name'], value='\n'.join(lines), inline=False)
|
||||||
embed.set_footer(text=f'Updates every {config.CHECK_INTERVAL}s')
|
embed.set_footer(text=self._footer_text(using_backup))
|
||||||
return embed
|
return embed
|
||||||
|
|
||||||
|
def _footer_text(self, using_backup):
|
||||||
|
text = f'Updates every {config.CHECK_INTERVAL}s'
|
||||||
|
if using_backup:
|
||||||
|
text += ' · ⚠ targets.json invalid — using last known-good config'
|
||||||
|
return text
|
||||||
|
|
||||||
async def update_embeds(self):
|
async def update_embeds(self):
|
||||||
if not self.status_messages:
|
if not self.status_messages:
|
||||||
return
|
return
|
||||||
results = await self.run_checks()
|
results, using_backup = await self.run_checks()
|
||||||
embed = self.build_embed(results)
|
embed = self.build_embed(results, using_backup)
|
||||||
|
|
||||||
kept = []
|
kept = []
|
||||||
for entry in self.status_messages:
|
for entry in self.status_messages:
|
||||||
@@ -148,8 +178,14 @@ bot = HealthBot()
|
|||||||
@bot.tree.command(name='status', description='Post a live-updating health embed for the services')
|
@bot.tree.command(name='status', description='Post a live-updating health embed for the services')
|
||||||
async def status(interaction: discord.Interaction):
|
async def status(interaction: discord.Interaction):
|
||||||
await interaction.response.defer()
|
await interaction.response.defer()
|
||||||
results = await bot.run_checks()
|
try:
|
||||||
embed = bot.build_embed(results)
|
results, using_backup = await bot.run_checks()
|
||||||
|
except checks.TargetsConfigError as exc:
|
||||||
|
await interaction.followup.send(
|
||||||
|
f'Could not load targets config: {exc}', ephemeral=True,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
embed = bot.build_embed(results, using_backup)
|
||||||
|
|
||||||
existing = next(
|
existing = next(
|
||||||
(m for m in bot.status_messages if m['channel_id'] == interaction.channel_id),
|
(m for m in bot.status_messages if m['channel_id'] == interaction.channel_id),
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
discord.py
|
discord.py
|
||||||
aiohttp
|
aiohttp
|
||||||
python-dotenv
|
python-dotenv
|
||||||
|
mcstatus==14.1.0
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "Gitea",
|
||||||
|
"url": "https://gitea.rainbow-herring.ts.net/api/healthz",
|
||||||
|
"kind": "gitea",
|
||||||
|
"critical": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Packs Site (Production)",
|
||||||
|
"url": "https://jakerasp.rainbow-herring.ts.net/healthz/",
|
||||||
|
"kind": "packs",
|
||||||
|
"critical": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Jake's Minecraft Server",
|
||||||
|
"host": "jakelaptop.rainbow-herring.ts.net",
|
||||||
|
"port": 10000,
|
||||||
|
"kind": "minecraft",
|
||||||
|
"critical": true
|
||||||
|
}
|
||||||
|
]
|
||||||
Reference in New Issue
Block a user