JakeBreath
03dd235f3a
Backend (new apps.follows): - FollowedTag/FollowedPool/FollowedPost models; per-user follows with unseen tracking, plus FollowCloud for the cached blacklist cloud. - Two periodic commands sharing one fetch path: sync_followed_tags and sync_followed_pools fetch each followed tag/pool's newest posts (one e621 search per unique follow), store unseen feed rows, refresh covers and pool metadata; both fall back to anonymous e621 access. - API: /api/follows/tags|pools (follow, unfollow, mark seen), a merged feed with per-follow filtering, and /api/follows/cloud/ which rebuilds the blacklisted-tag cloud in a daemon thread when its 10 min cache is stale (polling returns building/ready). - e621 client now supports anonymous reads; trimmed posts carry preview URLs for covers and feed tiles. Frontend: - /followed page: follow forms, cover cards with unseen badges and Mark seen, merged feed with filter/unseen toggle, and a blacklist cloud panel that polls while building. Followed nav entry added.
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
"""Blacklisted-tag cloud for the Followed page.
|
|
|
|
The cloud counts tags from the user's e621 blacklist that appear in their
|
|
followed feeds. Building it means one e621 request (the blacklist itself)
|
|
plus a local scan of stored feed posts, so it runs in a daemon thread and
|
|
the page polls until the fresh result lands.
|
|
"""
|
|
|
|
import logging
|
|
import threading
|
|
from datetime import timedelta
|
|
from fnmatch import fnmatch
|
|
|
|
from django.contrib.auth import get_user_model
|
|
from django.db import connection
|
|
from django.utils import timezone
|
|
|
|
from apps.library import e621, guest_filter
|
|
|
|
from .models import FollowCloud, FollowedPost
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CLOUD_TTL = timedelta(minutes=10)
|
|
BUILD_STALE = timedelta(minutes=5)
|
|
MAX_CLOUD_TAGS = 60
|
|
|
|
|
|
def blacklist_patterns_for(user):
|
|
"""(patterns, source): the user's e621 blacklist, else the guest default."""
|
|
if e621.credentials_configured(user):
|
|
try:
|
|
me = e621.get(user, "/users/me.json")
|
|
lines = str(me.get("blacklisted_tags") or "").splitlines()
|
|
patterns = guest_filter.parse_blacklist_patterns(lines)
|
|
if patterns:
|
|
return patterns, "user"
|
|
except e621.E621Error as exc:
|
|
logger.warning("Could not fetch the e621 blacklist for %s: %s", user, exc)
|
|
return guest_filter.guest_blacklist_patterns(), "default"
|
|
|
|
|
|
def compute_cloud(user):
|
|
"""Count blacklist matches across every stored feed post of one user."""
|
|
patterns, source = blacklist_patterns_for(user)
|
|
counts = {}
|
|
seen_posts = set()
|
|
rows = FollowedPost.objects.filter(user=user).values_list("post_id", "data")
|
|
for post_id, data in rows:
|
|
if post_id in seen_posts:
|
|
continue
|
|
seen_posts.add(post_id)
|
|
if not isinstance(data, dict):
|
|
continue
|
|
categories = data.get("tags")
|
|
if not isinstance(categories, dict):
|
|
continue
|
|
for values in categories.values():
|
|
if not isinstance(values, list):
|
|
continue
|
|
for tag in values:
|
|
name = str(tag).strip().lower()
|
|
if name and any(fnmatch(name, pattern) for pattern in patterns):
|
|
counts[name] = counts.get(name, 0) + 1
|
|
tags = sorted(counts.items(), key=lambda entry: (-entry[1], entry[0]))
|
|
return {
|
|
"tags": [[tag, count] for tag, count in tags[:MAX_CLOUD_TAGS]],
|
|
"posts": len(seen_posts),
|
|
"source": source,
|
|
"blacklist_count": len(patterns),
|
|
}
|
|
|
|
|
|
def start_cloud_build(user_id):
|
|
thread = threading.Thread(target=run_cloud_build, args=(user_id,), daemon=True)
|
|
thread.start()
|
|
|
|
|
|
def run_cloud_build(user_id):
|
|
User = get_user_model()
|
|
try:
|
|
user = User.objects.filter(id=user_id).first()
|
|
if user is None:
|
|
return
|
|
data = compute_cloud(user)
|
|
FollowCloud.objects.filter(user_id=user_id).update(
|
|
data=data, computed_at=timezone.now()
|
|
)
|
|
except Exception: # noqa: BLE001 - background builds must not crash workers
|
|
logger.exception("Blacklist cloud build failed for user %s", user_id)
|
|
finally:
|
|
FollowCloud.objects.filter(user_id=user_id).update(
|
|
building=False, building_since=None
|
|
)
|
|
# Background threads hold their own DB connection; release it so
|
|
# Gunicorn workers do not leak connections when threads finish.
|
|
connection.close()
|