diff --git a/AGENTS.md b/AGENTS.md index 106b88c..a56ae9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,9 @@ Project constraints (do not regress): Gitea registry with deploy/push_*.sh (multi-arch, :latest + :sha, GIT_HASH baked in for the version pill); deploy/gen_env.sh generates .env with openssl secrets (--update keeps SECRET_KEY, --force rotates it). +- Periodic commands (follow syncs, similarity cleanup, guest blacklist + refresh) run in the composes' `scheduler` service — the backend image with + the j621-scheduler entrypoint, intervals via J621_*_EVERY. No host cron. - Security/permission tests live in backend/apps/core/tests and need a one-time grant: GRANT ALL ON `test_j621`.* TO 'j621'@'%'; diff --git a/ROADMAP.md b/ROADMAP.md index 26e9c94..8bd2371 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -145,10 +145,12 @@ Files now stage first and are resolved before entering the library. ## 6. Infrastructure -- [ ] Guest blacklist refresh on a timer (in-container scheduler) -- [ ] Follow sync on a timer (in-container scheduler, e.g. every 30 minutes) -- [ ] Similarity temp cleanup on a timer (in-container scheduler; the TTL - also cleans lazily when new checks are created) +- [x] Guest blacklist refresh on a timer (the composes' `scheduler` service; + `J621_BLACKLIST_EVERY`, default daily) +- [x] Follow sync on a timer (same scheduler: `sync_followed_tags` + + `sync_followed_pools`, default every 30 minutes) +- [x] Similarity temp cleanup on a timer (same scheduler, default hourly; + the TTL also cleans lazily when new checks are created) - [x] Production setup: two Docker images (SPA on static nginx, API on gunicorn + whitenoise with ffmpeg) and three compose variants (both / frontend-only / backend-only) behind a shared nginx proxy service, each diff --git a/deploy/.env.example b/deploy/.env.example index cafa2f3..4fc7f04 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -49,6 +49,12 @@ DB_ROOT_PASSWORD=j621root # GUNICORN_THREADS=4 # GUNICORN_TIMEOUT=120 +# Scheduler intervals in seconds (the "scheduler" service runs the periodic +# management commands; see deploy/README.md) +# J621_SYNC_EVERY=1800 +# J621_CLEAN_EVERY=3600 +# J621_BLACKLIST_EVERY=86400 + # Rate limits (per IP anonymous, per account signed in) # THROTTLE_ANON=120/min # THROTTLE_USER=600/min diff --git a/deploy/J621-Backend b/deploy/J621-Backend index cfecb37..b336542 100644 --- a/deploy/J621-Backend +++ b/deploy/J621-Backend @@ -28,8 +28,9 @@ RUN pip install --no-cache-dir -r requirements.txt COPY backend/ ./ COPY deploy/backend-entrypoint.sh /usr/local/bin/j621-entrypoint +COPY deploy/scheduler-entrypoint.sh /usr/local/bin/j621-scheduler -RUN chmod +x /usr/local/bin/j621-entrypoint \ +RUN chmod +x /usr/local/bin/j621-entrypoint /usr/local/bin/j621-scheduler \ && mkdir -p /app/media /app/logs \ && python manage.py collectstatic --noinput diff --git a/deploy/README.md b/deploy/README.md index 231eec0..695ac64 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -123,6 +123,23 @@ Push multi-arch images to the Gitea registry: They tag `:latest` and `:` and expect `docker login gitea.rainbow-herring.ts.net` to succeed. +## Scheduled jobs + +Compose files with a backend also run a **`scheduler`** service — the same +backend image with a different entrypoint, so no host cron is involved: + +| Job | Default interval | Env override | +| --- | --- | --- | +| `sync_followed_tags` + `sync_followed_pools` | every 30 minutes | `J621_SYNC_EVERY` | +| `cleanup_similarity` | hourly | `J621_CLEAN_EVERY` | +| `refresh_guest_blacklist` | daily | `J621_BLACKLIST_EVERY` | + +It waits for the database and migrations before its first run, runs every job +once on start, then keeps to the intervals (failures are logged and retried +next round). Output goes to `docker compose logs scheduler`. Intervals are +seconds, set in `deploy/.env`. The frontend-only composes have no backend, so +no scheduler. + ## Tests The security/permission suite lives in `backend/apps/core/tests/`: diff --git a/deploy/compose.backend.yml b/deploy/compose.backend.yml index 55e0a42..6528fc6 100644 --- a/deploy/compose.backend.yml +++ b/deploy/compose.backend.yml @@ -68,6 +68,33 @@ services: redis: condition: service_healthy + # Periodic maintenance inside the deployment (no host cron): follow syncs, + # similarity cleanup and the guest blacklist refresh. + scheduler: + image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-backend:${J621_TAG:-latest} + build: + context: .. + dockerfile: deploy/J621-Backend + args: + GIT_HASH: ${GIT_HASH:-unknown} + restart: unless-stopped + entrypoint: ["j621-scheduler"] + env_file: [./.env] + environment: + DB_HOST: mariadb + DB_PORT: "3306" + REDIS_URL: redis://redis:6379/1 + SYNC_EVERY: ${J621_SYNC_EVERY:-1800} + CLEAN_EVERY: ${J621_CLEAN_EVERY:-3600} + BLACKLIST_EVERY: ${J621_BLACKLIST_EVERY:-86400} + volumes: + - ./data/media:/app/media + depends_on: + mariadb: + condition: service_healthy + redis: + condition: service_healthy + nginx: image: nginx:1.29-alpine restart: unless-stopped diff --git a/deploy/compose.tailnet.backend.yml b/deploy/compose.tailnet.backend.yml index de2dda4..4343971 100644 --- a/deploy/compose.tailnet.backend.yml +++ b/deploy/compose.tailnet.backend.yml @@ -68,6 +68,33 @@ services: redis: condition: service_healthy + # Periodic maintenance inside the deployment (no host cron): follow syncs, + # similarity cleanup and the guest blacklist refresh. + scheduler: + image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-backend:${J621_TAG:-latest} + build: + context: .. + dockerfile: deploy/J621-Backend + args: + GIT_HASH: ${GIT_HASH:-unknown} + restart: unless-stopped + entrypoint: ["j621-scheduler"] + env_file: [./.env] + environment: + DB_HOST: mariadb + DB_PORT: "3306" + REDIS_URL: redis://redis:6379/1 + SYNC_EVERY: ${J621_SYNC_EVERY:-1800} + CLEAN_EVERY: ${J621_CLEAN_EVERY:-3600} + BLACKLIST_EVERY: ${J621_BLACKLIST_EVERY:-86400} + volumes: + - ./data/media:/app/media + depends_on: + mariadb: + condition: service_healthy + redis: + condition: service_healthy + nginx: image: nginx:1.29-alpine restart: unless-stopped diff --git a/deploy/compose.tailnet.yml b/deploy/compose.tailnet.yml index 0e72f35..1db8bd8 100644 --- a/deploy/compose.tailnet.yml +++ b/deploy/compose.tailnet.yml @@ -67,6 +67,33 @@ services: redis: condition: service_healthy + # Periodic maintenance inside the deployment (no host cron): follow syncs, + # similarity cleanup and the guest blacklist refresh. + scheduler: + image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-backend:${J621_TAG:-latest} + build: + context: .. + dockerfile: deploy/J621-Backend + args: + GIT_HASH: ${GIT_HASH:-unknown} + restart: unless-stopped + entrypoint: ["j621-scheduler"] + env_file: [./.env] + environment: + DB_HOST: mariadb + DB_PORT: "3306" + REDIS_URL: redis://redis:6379/1 + SYNC_EVERY: ${J621_SYNC_EVERY:-1800} + CLEAN_EVERY: ${J621_CLEAN_EVERY:-3600} + BLACKLIST_EVERY: ${J621_BLACKLIST_EVERY:-86400} + volumes: + - ./data/media:/app/media + depends_on: + mariadb: + condition: service_healthy + redis: + condition: service_healthy + frontend: image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-frontend:${J621_TAG:-latest} build: diff --git a/deploy/compose.yml b/deploy/compose.yml index 6c177a5..8b9d9fb 100644 --- a/deploy/compose.yml +++ b/deploy/compose.yml @@ -66,6 +66,33 @@ services: redis: condition: service_healthy + # Periodic maintenance inside the deployment (no host cron): follow syncs, + # similarity cleanup and the guest blacklist refresh. + scheduler: + image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-backend:${J621_TAG:-latest} + build: + context: .. + dockerfile: deploy/J621-Backend + args: + GIT_HASH: ${GIT_HASH:-unknown} + restart: unless-stopped + entrypoint: ["j621-scheduler"] + env_file: [./.env] + environment: + DB_HOST: mariadb + DB_PORT: "3306" + REDIS_URL: redis://redis:6379/1 + SYNC_EVERY: ${J621_SYNC_EVERY:-1800} + CLEAN_EVERY: ${J621_CLEAN_EVERY:-3600} + BLACKLIST_EVERY: ${J621_BLACKLIST_EVERY:-86400} + volumes: + - ./data/media:/app/media + depends_on: + mariadb: + condition: service_healthy + redis: + condition: service_healthy + frontend: image: ${J621_REGISTRY:-gitea.rainbow-herring.ts.net/jakebreath}/j621-frontend:${J621_TAG:-latest} build: diff --git a/deploy/scheduler-entrypoint.sh b/deploy/scheduler-entrypoint.sh new file mode 100644 index 0000000..7afff23 --- /dev/null +++ b/deploy/scheduler-entrypoint.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Periodic maintenance for a J621 deployment (runs in the "scheduler" +# service; no host cron involved). +# +# sync_followed_tags + sync_followed_pools every SYNC_EVERY seconds (30 min) +# cleanup_similarity every CLEAN_EVERY seconds (1 h) +# refresh_guest_blacklist every BLACKLIST_EVERY (24 h) +# +# Waits for the database and migrations to be ready, runs everything once, +# then keeps checking. A failing command is logged and retried next interval, +# so a temporary e621 outage is not fatal. +set -u + +SYNC_EVERY="${SYNC_EVERY:-1800}" +CLEAN_EVERY="${CLEAN_EVERY:-3600}" +BLACKLIST_EVERY="${BLACKLIST_EVERY:-86400}" + +log() { echo "[scheduler] $(date -Iseconds) $*"; } + +run() { + log "running: $*" + if "$@"; then + log "done: $*" + else + log "FAILED (retrying at the next interval): $*" + fi +} + +command -v python >/dev/null || { log "python not found"; exit 1; } + +log "waiting for the database and migrations..." +until python manage.py migrate --check >/dev/null 2>&1; do + sleep 10 +done +log "database ready; intervals: sync=${SYNC_EVERY}s cleanup=${CLEAN_EVERY}s blacklist=${BLACKLIST_EVERY}s" + +now=$(date +%s) +last_sync=$((now - SYNC_EVERY)) +last_clean=$((now - CLEAN_EVERY)) +last_blacklist=$((now - BLACKLIST_EVERY)) + +while true; do + now=$(date +%s) + + if [ $((now - last_sync)) -ge "$SYNC_EVERY" ]; then + last_sync=$now + run python manage.py sync_followed_tags + run python manage.py sync_followed_pools + fi + + if [ $((now - last_clean)) -ge "$CLEAN_EVERY" ]; then + last_clean=$now + run python manage.py cleanup_similarity + fi + + if [ $((now - last_blacklist)) -ge "$BLACKLIST_EVERY" ]; then + last_blacklist=$now + run python manage.py refresh_guest_blacklist + fi + + sleep 30 +done