Follows: followed tags/pools, feeds, unseen badges and blacklist cloud
Backend (new apps.follows): - FollowedTag/FollowedPool/FollowedPost models; per-user follows with unseen tracking, plus FollowCloud for the cached blacklist cloud. - Two periodic commands sharing one fetch path: sync_followed_tags and sync_followed_pools fetch each followed tag/pool's newest posts (one e621 search per unique follow), store unseen feed rows, refresh covers and pool metadata; both fall back to anonymous e621 access. - API: /api/follows/tags|pools (follow, unfollow, mark seen), a merged feed with per-follow filtering, and /api/follows/cloud/ which rebuilds the blacklisted-tag cloud in a daemon thread when its 10 min cache is stale (polling returns building/ready). - e621 client now supports anonymous reads; trimmed posts carry preview URLs for covers and feed tiles. Frontend: - /followed page: follow forms, cover cards with unseen badges and Mark seen, merged feed with filter/unseen toggle, and a blacklist cloud panel that polls while building. Followed nav entry added.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class FollowsConfig(AppConfig):
|
||||
name = "apps.follows"
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Blacklisted-tag cloud for the Followed page.
|
||||
|
||||
The cloud counts tags from the user's e621 blacklist that appear in their
|
||||
followed feeds. Building it means one e621 request (the blacklist itself)
|
||||
plus a local scan of stored feed posts, so it runs in a daemon thread and
|
||||
the page polls until the fresh result lands.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
from fnmatch import fnmatch
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import connection
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.library import e621, guest_filter
|
||||
|
||||
from .models import FollowCloud, FollowedPost
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CLOUD_TTL = timedelta(minutes=10)
|
||||
BUILD_STALE = timedelta(minutes=5)
|
||||
MAX_CLOUD_TAGS = 60
|
||||
|
||||
|
||||
def blacklist_patterns_for(user):
|
||||
"""(patterns, source): the user's e621 blacklist, else the guest default."""
|
||||
if e621.credentials_configured(user):
|
||||
try:
|
||||
me = e621.get(user, "/users/me.json")
|
||||
lines = str(me.get("blacklisted_tags") or "").splitlines()
|
||||
patterns = guest_filter.parse_blacklist_patterns(lines)
|
||||
if patterns:
|
||||
return patterns, "user"
|
||||
except e621.E621Error as exc:
|
||||
logger.warning("Could not fetch the e621 blacklist for %s: %s", user, exc)
|
||||
return guest_filter.guest_blacklist_patterns(), "default"
|
||||
|
||||
|
||||
def compute_cloud(user):
|
||||
"""Count blacklist matches across every stored feed post of one user."""
|
||||
patterns, source = blacklist_patterns_for(user)
|
||||
counts = {}
|
||||
seen_posts = set()
|
||||
rows = FollowedPost.objects.filter(user=user).values_list("post_id", "data")
|
||||
for post_id, data in rows:
|
||||
if post_id in seen_posts:
|
||||
continue
|
||||
seen_posts.add(post_id)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
categories = data.get("tags")
|
||||
if not isinstance(categories, dict):
|
||||
continue
|
||||
for values in categories.values():
|
||||
if not isinstance(values, list):
|
||||
continue
|
||||
for tag in values:
|
||||
name = str(tag).strip().lower()
|
||||
if name and any(fnmatch(name, pattern) for pattern in patterns):
|
||||
counts[name] = counts.get(name, 0) + 1
|
||||
tags = sorted(counts.items(), key=lambda entry: (-entry[1], entry[0]))
|
||||
return {
|
||||
"tags": [[tag, count] for tag, count in tags[:MAX_CLOUD_TAGS]],
|
||||
"posts": len(seen_posts),
|
||||
"source": source,
|
||||
"blacklist_count": len(patterns),
|
||||
}
|
||||
|
||||
|
||||
def start_cloud_build(user_id):
|
||||
thread = threading.Thread(target=run_cloud_build, args=(user_id,), daemon=True)
|
||||
thread.start()
|
||||
|
||||
|
||||
def run_cloud_build(user_id):
|
||||
User = get_user_model()
|
||||
try:
|
||||
user = User.objects.filter(id=user_id).first()
|
||||
if user is None:
|
||||
return
|
||||
data = compute_cloud(user)
|
||||
FollowCloud.objects.filter(user_id=user_id).update(
|
||||
data=data, computed_at=timezone.now()
|
||||
)
|
||||
except Exception: # noqa: BLE001 - background builds must not crash workers
|
||||
logger.exception("Blacklist cloud build failed for user %s", user_id)
|
||||
finally:
|
||||
FollowCloud.objects.filter(user_id=user_id).update(
|
||||
building=False, building_since=None
|
||||
)
|
||||
# Background threads hold their own DB connection; release it so
|
||||
# Gunicorn workers do not leak connections when threads finish.
|
||||
connection.close()
|
||||
@@ -0,0 +1,62 @@
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.follows import sync
|
||||
from apps.follows.models import FollowedPool
|
||||
from apps.library import e621
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Fetch new posts for every followed pool: fills each feed, refreshes "
|
||||
"the card cover (latest image), pool name/count and unseen badges."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--user",
|
||||
help="Only sync follows belonging to this username "
|
||||
"(default: every user's follows).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=sync.FEED_LIMIT,
|
||||
help=f"Posts fetched per pool (default {sync.FEED_LIMIT}).",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
username = options.get("user")
|
||||
if username and sync.preferred_fetch_user(username) is None:
|
||||
raise CommandError(f"No such user: {username}")
|
||||
fetch_user = sync.preferred_fetch_user(username)
|
||||
|
||||
follows = FollowedPool.objects.select_related("user").order_by(
|
||||
"pool_id", "user_id"
|
||||
)
|
||||
if username:
|
||||
follows = follows.filter(user__username=username)
|
||||
follows = list(follows)
|
||||
grouped = {}
|
||||
for follow in follows:
|
||||
grouped.setdefault(follow.pool_id, []).append(follow)
|
||||
|
||||
created = 0
|
||||
for pool_id, group in grouped.items():
|
||||
try:
|
||||
posts = sync.search_posts(
|
||||
fetch_user, f"pool:{pool_id}", options["limit"]
|
||||
)
|
||||
except e621.E621Error as exc:
|
||||
raise CommandError(f"Aborted at pool #{pool_id}: {exc}")
|
||||
for follow in group:
|
||||
created += sync.sync_pool_follow(
|
||||
follow, fetch_user, posts=posts
|
||||
)
|
||||
|
||||
identity = getattr(fetch_user, "username", None) or "anonymous"
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f"Synced {len(grouped)} pool(s) across {len(follows)} follow(s) "
|
||||
f"as {identity}: {created} new post(s)."
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.follows import sync
|
||||
from apps.follows.models import FollowedTag
|
||||
from apps.library import e621
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Fetch new posts for every followed tag: fills each feed, refreshes "
|
||||
"the card cover (latest image) and unseen badges."
|
||||
)
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--user",
|
||||
help="Only sync follows belonging to this username "
|
||||
"(default: every user's follows).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=sync.FEED_LIMIT,
|
||||
help=f"Posts fetched per tag (default {sync.FEED_LIMIT}).",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
username = options.get("user")
|
||||
if username and sync.preferred_fetch_user(username) is None:
|
||||
raise CommandError(f"No such user: {username}")
|
||||
fetch_user = sync.preferred_fetch_user(username)
|
||||
|
||||
follows = FollowedTag.objects.select_related("user").order_by(
|
||||
"tag", "user_id"
|
||||
)
|
||||
if username:
|
||||
follows = follows.filter(user__username=username)
|
||||
follows = list(follows)
|
||||
grouped = {}
|
||||
for follow in follows:
|
||||
grouped.setdefault(follow.tag, []).append(follow)
|
||||
|
||||
created = 0
|
||||
for tag, group in grouped.items():
|
||||
try:
|
||||
posts = sync.search_posts(fetch_user, tag, options["limit"])
|
||||
except e621.E621Error as exc:
|
||||
raise CommandError(f"Aborted at tag '{tag}': {exc}")
|
||||
for follow in group:
|
||||
created += sync.sync_tag_follow(
|
||||
follow, fetch_user, posts=posts
|
||||
)
|
||||
|
||||
identity = getattr(fetch_user, "username", None) or "anonymous"
|
||||
self.stdout.write(
|
||||
self.style.SUCCESS(
|
||||
f"Synced {len(grouped)} tag(s) across {len(follows)} follow(s) "
|
||||
f"as {identity}: {created} new post(s)."
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,94 @@
|
||||
# Generated by Django 6.1.1 on 2026-09-17 19:04
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='FollowCloud',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('data', models.JSONField(blank=True, null=True)),
|
||||
('computed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('building', models.BooleanField(default=False)),
|
||||
('building_since', models.DateTimeField(blank=True, null=True)),
|
||||
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='follow_cloud', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='FollowedPool',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('pool_id', models.IntegerField()),
|
||||
('name', models.CharField(blank=True, default='', max_length=200)),
|
||||
('post_count', models.IntegerField(default=0)),
|
||||
('cover_data', models.JSONField(blank=True, null=True)),
|
||||
('last_synced_at', models.DateTimeField(blank=True, null=True)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='followed_pools', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['pool_id'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='FollowedTag',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('tag', models.CharField(max_length=200)),
|
||||
('cover_data', models.JSONField(blank=True, null=True)),
|
||||
('last_synced_at', models.DateTimeField(blank=True, null=True)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='followed_tags', to=settings.AUTH_USER_MODEL)),
|
||||
],
|
||||
options={
|
||||
'ordering': ['tag'],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='FollowedPost',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('post_id', models.IntegerField(db_index=True)),
|
||||
('data', models.JSONField(default=dict)),
|
||||
('seen', models.BooleanField(db_index=True, default=False)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('pool', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='posts', to='follows.followedpool')),
|
||||
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='followed_posts', to=settings.AUTH_USER_MODEL)),
|
||||
('tag', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='posts', to='follows.followedtag')),
|
||||
],
|
||||
options={
|
||||
'ordering': ['-post_id'],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='followedpool',
|
||||
constraint=models.UniqueConstraint(fields=('user', 'pool_id'), name='uniq_followed_pool_per_user'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='followedtag',
|
||||
constraint=models.UniqueConstraint(fields=('user', 'tag'), name='uniq_followed_tag_per_user'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='followedpost',
|
||||
index=models.Index(fields=['user', 'post_id'], name='follows_fol_user_id_47fdab_idx'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='followedpost',
|
||||
constraint=models.UniqueConstraint(fields=('tag', 'post_id'), name='uniq_followed_tag_post'),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name='followedpost',
|
||||
constraint=models.UniqueConstraint(fields=('pool', 'post_id'), name='uniq_followed_pool_post'),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,114 @@
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
|
||||
class FollowedTag(models.Model):
|
||||
"""A tag one user subscribes to; feeds are filled by the sync command."""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="followed_tags",
|
||||
)
|
||||
tag = models.CharField(max_length=200)
|
||||
cover_data = models.JSONField(null=True, blank=True)
|
||||
last_synced_at = models.DateTimeField(null=True, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["tag"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["user", "tag"], name="uniq_followed_tag_per_user"
|
||||
),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.user} follows tag:{self.tag}"
|
||||
|
||||
|
||||
class FollowedPool(models.Model):
|
||||
"""A pool one user subscribes to; feeds are filled by the sync command."""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="followed_pools",
|
||||
)
|
||||
pool_id = models.IntegerField()
|
||||
name = models.CharField(max_length=200, blank=True, default="")
|
||||
post_count = models.IntegerField(default=0)
|
||||
cover_data = models.JSONField(null=True, blank=True)
|
||||
last_synced_at = models.DateTimeField(null=True, blank=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["pool_id"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["user", "pool_id"], name="uniq_followed_pool_per_user"
|
||||
),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.user} follows pool:{self.pool_id}"
|
||||
|
||||
|
||||
class FollowedPost(models.Model):
|
||||
"""One post seen in a followed feed (stored once per follow source)."""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="followed_posts",
|
||||
)
|
||||
tag = models.ForeignKey(
|
||||
FollowedTag,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="posts",
|
||||
)
|
||||
pool = models.ForeignKey(
|
||||
FollowedPool,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="posts",
|
||||
)
|
||||
post_id = models.IntegerField(db_index=True)
|
||||
data = models.JSONField(default=dict)
|
||||
seen = models.BooleanField(default=False, db_index=True)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-post_id"]
|
||||
indexes = [models.Index(fields=["user", "post_id"])]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["tag", "post_id"], name="uniq_followed_tag_post"
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["pool", "post_id"], name="uniq_followed_pool_post"
|
||||
),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"#{self.post_id} for {self.user}"
|
||||
|
||||
|
||||
class FollowCloud(models.Model):
|
||||
"""Cached blacklisted-tag cloud for the Followed page, built in a thread."""
|
||||
|
||||
user = models.OneToOneField(
|
||||
settings.AUTH_USER_MODEL,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="follow_cloud",
|
||||
)
|
||||
data = models.JSONField(null=True, blank=True)
|
||||
computed_at = models.DateTimeField(null=True, blank=True)
|
||||
building = models.BooleanField(default=False)
|
||||
building_since = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
def __str__(self):
|
||||
return f"Follow cloud for {self.user}"
|
||||
@@ -0,0 +1,53 @@
|
||||
from rest_framework import serializers
|
||||
|
||||
from .models import FollowedPool, FollowedTag
|
||||
|
||||
|
||||
class FollowedTagSerializer(serializers.ModelSerializer):
|
||||
unseen_count = serializers.IntegerField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = FollowedTag
|
||||
fields = [
|
||||
"id",
|
||||
"tag",
|
||||
"unseen_count",
|
||||
"cover_data",
|
||||
"last_synced_at",
|
||||
"created_at",
|
||||
]
|
||||
read_only_fields = [
|
||||
"id",
|
||||
"tag",
|
||||
"unseen_count",
|
||||
"cover_data",
|
||||
"last_synced_at",
|
||||
"created_at",
|
||||
]
|
||||
|
||||
|
||||
class FollowedPoolSerializer(serializers.ModelSerializer):
|
||||
unseen_count = serializers.IntegerField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = FollowedPool
|
||||
fields = [
|
||||
"id",
|
||||
"pool_id",
|
||||
"name",
|
||||
"post_count",
|
||||
"unseen_count",
|
||||
"cover_data",
|
||||
"last_synced_at",
|
||||
"created_at",
|
||||
]
|
||||
read_only_fields = [
|
||||
"id",
|
||||
"pool_id",
|
||||
"name",
|
||||
"post_count",
|
||||
"unseen_count",
|
||||
"cover_data",
|
||||
"last_synced_at",
|
||||
"created_at",
|
||||
]
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Fetch and store followed tag/pool feeds from e621.
|
||||
|
||||
Used by the two periodic management commands (`sync_followed_tags`,
|
||||
`sync_followed_pools`) and by the follow API for the first fill of a new
|
||||
subscription. One e621 search per unique tag/pool is shared across every
|
||||
user following it.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.library import e621
|
||||
from apps.library.services import trim_e621_post
|
||||
|
||||
from .models import FollowedPool, FollowedPost, FollowedTag
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FEED_LIMIT = 24
|
||||
PRUNE_KEEP = 200
|
||||
|
||||
|
||||
def preferred_fetch_user(username=None):
|
||||
"""A configured account for sync calls; None means run anonymously.
|
||||
|
||||
e621 serves post searches and pool reads anonymously, so a missing
|
||||
account is not fatal — credentials mainly raise rate limits.
|
||||
"""
|
||||
User = get_user_model()
|
||||
if username:
|
||||
return User.objects.filter(username=username).first()
|
||||
return (
|
||||
User.objects.filter(is_staff=True)
|
||||
.exclude(e621_username="")
|
||||
.exclude(e621_api_key="")
|
||||
.order_by("id")
|
||||
.first()
|
||||
)
|
||||
|
||||
|
||||
def search_posts(user, term, limit=FEED_LIMIT):
|
||||
"""Newest posts for a search term (``tag`` or ``pool:<id>``)."""
|
||||
payload = e621.get(
|
||||
user,
|
||||
"/posts.json",
|
||||
params={"tags": term, "limit": limit},
|
||||
require_auth=False,
|
||||
)
|
||||
posts = payload.get("posts") if isinstance(payload, dict) else None
|
||||
return posts or []
|
||||
|
||||
|
||||
def cover_payload(post):
|
||||
"""Lightweight card cover: enough to render a thumbnail and rating."""
|
||||
if not isinstance(post, dict):
|
||||
return None
|
||||
preview = post.get("preview") or {}
|
||||
score = post.get("score") or {}
|
||||
return {
|
||||
"post_id": post.get("id"),
|
||||
"rating": post.get("rating"),
|
||||
"preview_url": preview.get("url"),
|
||||
"score": score.get("total"),
|
||||
"fav_count": post.get("fav_count"),
|
||||
}
|
||||
|
||||
|
||||
def feed_payload(post):
|
||||
"""Compact stored post for the feed (tags included for the tag cloud)."""
|
||||
trimmed = trim_e621_post(post)
|
||||
if trimmed is None:
|
||||
return None
|
||||
return {
|
||||
"id": trimmed.get("id"),
|
||||
"created_at": trimmed.get("created_at"),
|
||||
"rating": trimmed.get("rating"),
|
||||
"tags": trimmed.get("tags") or {},
|
||||
"score": trimmed.get("score") or {},
|
||||
"fav_count": trimmed.get("fav_count"),
|
||||
"file": trimmed.get("file") or {},
|
||||
"preview": trimmed.get("preview") or {},
|
||||
"uploader_name": trimmed.get("uploader_name"),
|
||||
}
|
||||
|
||||
|
||||
def _store_posts(follow, posts, tag=None, pool=None):
|
||||
"""Insert unseen rows for new posts. Returns how many were added."""
|
||||
existing = set(follow.posts.values_list("post_id", flat=True))
|
||||
rows = []
|
||||
for post in posts:
|
||||
post_id = post.get("id") if isinstance(post, dict) else None
|
||||
if not isinstance(post_id, int) or post_id in existing:
|
||||
continue
|
||||
data = feed_payload(post)
|
||||
if data is None:
|
||||
continue
|
||||
rows.append(
|
||||
FollowedPost(
|
||||
user=follow.user,
|
||||
post_id=post_id,
|
||||
data=data,
|
||||
tag=tag,
|
||||
pool=pool,
|
||||
)
|
||||
)
|
||||
if rows:
|
||||
FollowedPost.objects.bulk_create(rows, ignore_conflicts=True)
|
||||
_prune(follow)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _prune(follow):
|
||||
"""Keep only the newest PRUNE_KEEP posts per follow."""
|
||||
keep = list(
|
||||
follow.posts.order_by("-post_id").values_list("id", flat=True)[
|
||||
:PRUNE_KEEP
|
||||
]
|
||||
)
|
||||
follow.posts.exclude(id__in=keep).delete()
|
||||
|
||||
|
||||
def sync_tag_follow(follow: FollowedTag, fetch_user=None, limit=FEED_LIMIT, posts=None):
|
||||
"""Fetch new posts for one followed tag; updates cover and sync time."""
|
||||
if posts is None:
|
||||
posts = search_posts(fetch_user, follow.tag, limit)
|
||||
added = _store_posts(follow, posts, tag=follow)
|
||||
updates = ["last_synced_at"]
|
||||
cover = cover_payload(posts[0]) if posts else None
|
||||
if cover is not None:
|
||||
follow.cover_data = cover
|
||||
updates.append("cover_data")
|
||||
follow.last_synced_at = timezone.now()
|
||||
follow.save(update_fields=updates)
|
||||
return added
|
||||
|
||||
|
||||
def sync_pool_follow(
|
||||
follow: FollowedPool,
|
||||
fetch_user=None,
|
||||
limit=FEED_LIMIT,
|
||||
posts=None,
|
||||
refresh_meta=True,
|
||||
):
|
||||
"""Fetch new posts for one followed pool; updates cover and metadata."""
|
||||
if posts is None:
|
||||
posts = search_posts(fetch_user, f"pool:{follow.pool_id}", limit)
|
||||
added = _store_posts(follow, posts, pool=follow)
|
||||
updates = ["last_synced_at"]
|
||||
cover = cover_payload(posts[0]) if posts else None
|
||||
if cover is not None:
|
||||
follow.cover_data = cover
|
||||
updates.append("cover_data")
|
||||
if refresh_meta or not follow.name:
|
||||
try:
|
||||
pool = e621.get(
|
||||
fetch_user,
|
||||
f"/pools/{follow.pool_id}.json",
|
||||
require_auth=False,
|
||||
)
|
||||
except e621.E621Error as exc:
|
||||
logger.warning("Could not refresh pool %s: %s", follow.pool_id, exc)
|
||||
else:
|
||||
if isinstance(pool, dict) and pool.get("id"):
|
||||
follow.name = str(pool.get("name") or "")[:200]
|
||||
follow.post_count = int(pool.get("post_count") or 0)
|
||||
updates += ["name", "post_count"]
|
||||
follow.last_synced_at = timezone.now()
|
||||
follow.save(update_fields=updates)
|
||||
return added
|
||||
@@ -0,0 +1,25 @@
|
||||
from django.urls import include, path
|
||||
from rest_framework.routers import DefaultRouter
|
||||
|
||||
from .views import (
|
||||
FollowCloudView,
|
||||
FollowedFeedView,
|
||||
FollowedPoolViewSet,
|
||||
FollowedPostSeenView,
|
||||
FollowedTagViewSet,
|
||||
)
|
||||
|
||||
router = DefaultRouter()
|
||||
router.register("follows/tags", FollowedTagViewSet, basename="followed-tag")
|
||||
router.register("follows/pools", FollowedPoolViewSet, basename="followed-pool")
|
||||
|
||||
urlpatterns = [
|
||||
path("", include(router.urls)),
|
||||
path("follows/feed/", FollowedFeedView.as_view(), name="followed_feed"),
|
||||
path("follows/cloud/", FollowCloudView.as_view(), name="followed_cloud"),
|
||||
path(
|
||||
"follows/posts/<int:post_id>/seen/",
|
||||
FollowedPostSeenView.as_view(),
|
||||
name="followed_post_seen",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,317 @@
|
||||
import re
|
||||
|
||||
from django.db.models import Count, Q
|
||||
from django.utils import timezone
|
||||
from rest_framework import mixins, status, viewsets
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.library import e621
|
||||
|
||||
from . import cloud as cloud_service
|
||||
from . import sync
|
||||
from .models import FollowCloud, FollowedPool, FollowedPost, FollowedTag
|
||||
from .serializers import FollowedPoolSerializer, FollowedTagSerializer
|
||||
|
||||
POOL_URL_RE = re.compile(r"/pools/(\d+)")
|
||||
|
||||
|
||||
def _parse_page(request, default_per_page=48):
|
||||
try:
|
||||
per_page = int(request.query_params.get("per_page", default_per_page))
|
||||
except ValueError:
|
||||
per_page = default_per_page
|
||||
try:
|
||||
page = int(request.query_params.get("page", 1))
|
||||
except ValueError:
|
||||
page = 1
|
||||
return max(1, page), min(200, max(1, per_page))
|
||||
|
||||
|
||||
class FollowedTagViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.CreateModelMixin,
|
||||
mixins.DestroyModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
"""Tags the current user follows."""
|
||||
|
||||
serializer_class = FollowedTagSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
http_method_names = ["get", "post", "delete", "head", "options"]
|
||||
|
||||
def get_queryset(self):
|
||||
return (
|
||||
FollowedTag.objects.filter(user=self.request.user)
|
||||
.annotate(
|
||||
unseen_count=Count("posts", filter=Q(posts__seen=False), distinct=True)
|
||||
)
|
||||
.order_by("tag")
|
||||
)
|
||||
|
||||
def create(self, request):
|
||||
tag = str(request.data.get("tag") or "").strip().lower()[:200]
|
||||
if not tag:
|
||||
return Response(
|
||||
{"detail": "A tag is required."}, status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
if FollowedTag.objects.filter(user=request.user, tag=tag).exists():
|
||||
return Response(
|
||||
{"detail": f"Already following '{tag}'."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
try:
|
||||
results = e621.get(
|
||||
request.user,
|
||||
"/tags.json",
|
||||
params={"search[name_matches]": tag, "limit": 5},
|
||||
require_auth=False,
|
||||
)
|
||||
except e621.E621Error as exc:
|
||||
return Response(
|
||||
{"detail": str(exc)}, status=status.HTTP_502_BAD_GATEWAY
|
||||
)
|
||||
names = {
|
||||
str(entry.get("name", "")).lower()
|
||||
for entry in results
|
||||
if isinstance(entry, dict)
|
||||
}
|
||||
if tag not in names:
|
||||
return Response(
|
||||
{"detail": f"e621 has no tag named '{tag}'."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
follow = FollowedTag.objects.create(user=request.user, tag=tag)
|
||||
FollowCloud.objects.filter(user=request.user).update(computed_at=None)
|
||||
try:
|
||||
sync.sync_tag_follow(follow, request.user)
|
||||
except e621.E621Error:
|
||||
# The periodic command will catch up; the follow still stands.
|
||||
pass
|
||||
follow = self.get_queryset().get(pk=follow.pk)
|
||||
return Response(
|
||||
self.get_serializer(follow).data, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
def perform_destroy(self, instance):
|
||||
instance.delete()
|
||||
FollowCloud.objects.filter(user=self.request.user).update(computed_at=None)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def seen(self, request, pk=None):
|
||||
follow = self.get_object()
|
||||
follow.posts.update(seen=True)
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class FollowedPoolViewSet(
|
||||
mixins.ListModelMixin,
|
||||
mixins.CreateModelMixin,
|
||||
mixins.DestroyModelMixin,
|
||||
viewsets.GenericViewSet,
|
||||
):
|
||||
"""Pools the current user follows."""
|
||||
|
||||
serializer_class = FollowedPoolSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
http_method_names = ["get", "post", "delete", "head", "options"]
|
||||
|
||||
def get_queryset(self):
|
||||
return (
|
||||
FollowedPool.objects.filter(user=self.request.user)
|
||||
.annotate(
|
||||
unseen_count=Count("posts", filter=Q(posts__seen=False), distinct=True)
|
||||
)
|
||||
.order_by("pool_id")
|
||||
)
|
||||
|
||||
def create(self, request):
|
||||
raw = str(
|
||||
request.data.get("pool") or request.data.get("pool_id") or ""
|
||||
).strip()
|
||||
match = POOL_URL_RE.search(raw)
|
||||
if match:
|
||||
pool_id = int(match.group(1))
|
||||
elif raw.isdigit():
|
||||
pool_id = int(raw)
|
||||
else:
|
||||
return Response(
|
||||
{"detail": "Enter an e621 pool ID or pool URL."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if FollowedPool.objects.filter(user=request.user, pool_id=pool_id).exists():
|
||||
return Response(
|
||||
{"detail": f"Already following pool #{pool_id}."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
try:
|
||||
pool = e621.get(
|
||||
request.user, f"/pools/{pool_id}.json", require_auth=False
|
||||
)
|
||||
except e621.E621NotFound:
|
||||
return Response(
|
||||
{"detail": f"e621 has no pool #{pool_id}."},
|
||||
status=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
except e621.E621Error as exc:
|
||||
return Response(
|
||||
{"detail": str(exc)}, status=status.HTTP_502_BAD_GATEWAY
|
||||
)
|
||||
follow = FollowedPool.objects.create(
|
||||
user=request.user,
|
||||
pool_id=pool_id,
|
||||
name=str(pool.get("name") or "")[:200],
|
||||
post_count=int(pool.get("post_count") or 0),
|
||||
)
|
||||
FollowCloud.objects.filter(user=request.user).update(computed_at=None)
|
||||
try:
|
||||
sync.sync_pool_follow(follow, request.user, refresh_meta=False)
|
||||
except e621.E621Error:
|
||||
pass
|
||||
follow = self.get_queryset().get(pk=follow.pk)
|
||||
return Response(
|
||||
self.get_serializer(follow).data, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
def perform_destroy(self, instance):
|
||||
instance.delete()
|
||||
FollowCloud.objects.filter(user=self.request.user).update(computed_at=None)
|
||||
|
||||
@action(detail=True, methods=["post"])
|
||||
def seen(self, request, pk=None):
|
||||
follow = self.get_object()
|
||||
follow.posts.update(seen=True)
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class FollowedFeedView(APIView):
|
||||
"""Merged newest-first feed across followed tags and pools."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get(self, request):
|
||||
page, per_page = _parse_page(request)
|
||||
queryset = FollowedPost.objects.filter(user=request.user).order_by(
|
||||
"-post_id"
|
||||
)
|
||||
tag_id = request.query_params.get("tag", "")
|
||||
pool_id = request.query_params.get("pool", "")
|
||||
if tag_id.isdigit():
|
||||
queryset = queryset.filter(tag_id=int(tag_id))
|
||||
if pool_id.isdigit():
|
||||
queryset = queryset.filter(pool_id=int(pool_id))
|
||||
if request.query_params.get("unseen") == "1":
|
||||
queryset = queryset.filter(seen=False)
|
||||
|
||||
tag_names = dict(
|
||||
FollowedTag.objects.filter(user=request.user).values_list("id", "tag")
|
||||
)
|
||||
pool_names = dict(
|
||||
FollowedPool.objects.filter(user=request.user).values_list(
|
||||
"id", "name"
|
||||
)
|
||||
)
|
||||
entries = {}
|
||||
for post_id, seen, data, tag, pool in queryset.values_list(
|
||||
"post_id", "seen", "data", "tag_id", "pool_id"
|
||||
)[:2000]:
|
||||
entry = entries.get(post_id)
|
||||
if entry is None:
|
||||
entry = entries[post_id] = {
|
||||
"post_id": post_id,
|
||||
"seen": seen,
|
||||
"data": data,
|
||||
"sources": [],
|
||||
}
|
||||
else:
|
||||
entry["seen"] = entry["seen"] and seen
|
||||
if tag and not any(
|
||||
source["kind"] == "tag" and source["id"] == tag
|
||||
for source in entry["sources"]
|
||||
):
|
||||
entry["sources"].append(
|
||||
{
|
||||
"kind": "tag",
|
||||
"id": tag,
|
||||
"label": tag_names.get(tag, f"tag:{tag}"),
|
||||
}
|
||||
)
|
||||
if pool and not any(
|
||||
source["kind"] == "pool" and source["id"] == pool
|
||||
for source in entry["sources"]
|
||||
):
|
||||
entry["sources"].append(
|
||||
{
|
||||
"kind": "pool",
|
||||
"id": pool,
|
||||
"label": pool_names.get(pool) or f"Pool #{pool}",
|
||||
}
|
||||
)
|
||||
|
||||
items = list(entries.values())
|
||||
count = len(items)
|
||||
start = (page - 1) * per_page
|
||||
return Response(
|
||||
{
|
||||
"count": count,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"has_next": start + per_page < count,
|
||||
"results": items[start : start + per_page],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class FollowedPostSeenView(APIView):
|
||||
"""Mark one post seen across every follow it appears in."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def post(self, request, post_id):
|
||||
FollowedPost.objects.filter(user=request.user, post_id=post_id).update(
|
||||
seen=True
|
||||
)
|
||||
return Response({"success": True})
|
||||
|
||||
|
||||
class FollowCloudView(APIView):
|
||||
"""The blacklisted-tag cloud, rebuilt in a thread when stale."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get(self, request):
|
||||
cloud, _ = FollowCloud.objects.get_or_create(user=request.user)
|
||||
now = timezone.now()
|
||||
stale = cloud.computed_at is None or now - cloud.computed_at > (
|
||||
cloud_service.CLOUD_TTL
|
||||
)
|
||||
if stale:
|
||||
reclaim = Q(building=False) | Q(
|
||||
building_since__lt=now - cloud_service.BUILD_STALE
|
||||
)
|
||||
started = (
|
||||
FollowCloud.objects.filter(pk=cloud.pk)
|
||||
.filter(reclaim)
|
||||
.update(building=True, building_since=now)
|
||||
)
|
||||
if started:
|
||||
cloud_service.start_cloud_build(request.user.id)
|
||||
cloud.refresh_from_db()
|
||||
building = cloud.building and not (
|
||||
cloud.building_since
|
||||
and now - cloud.building_since > cloud_service.BUILD_STALE
|
||||
)
|
||||
data = cloud.data or {}
|
||||
return Response(
|
||||
{
|
||||
"status": "building" if building else "ready",
|
||||
"stale": stale,
|
||||
"tags": data.get("tags", []),
|
||||
"posts": data.get("posts", 0),
|
||||
"source": data.get("source"),
|
||||
"blacklist_count": data.get("blacklist_count", 0),
|
||||
"computed_at": cloud.computed_at,
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user