JakeBreath
09405d1a0f
- MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621.
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Minimal e621 API client for server-side matching and metadata refresh.
|
|
|
|
The SPA talks to e621 directly for browsing; this client exists for work the
|
|
browser cannot do reliably: long batch scans, and requests tied to a library
|
|
item rather than an open page. It uses the requesting user's stored
|
|
credentials and a global throttle (e621 asks for at most two requests per
|
|
second).
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
|
|
REQUEST_INTERVAL = 0.5 # seconds between requests, per process
|
|
|
|
|
|
class E621Error(Exception):
|
|
"""A non-404 failure while talking to e621."""
|
|
|
|
|
|
class E621NotFound(E621Error):
|
|
"""The requested post does not exist (HTTP 404)."""
|
|
|
|
|
|
_throttle_lock = threading.Lock()
|
|
_last_request_at = 0.0
|
|
|
|
|
|
def credentials_configured(user):
|
|
return bool(user is not None and getattr(user, "e621_configured", False))
|
|
|
|
|
|
def _wait_for_slot():
|
|
global _last_request_at
|
|
with _throttle_lock:
|
|
delay = _last_request_at + REQUEST_INTERVAL - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
_last_request_at = time.monotonic()
|
|
|
|
|
|
def get(user, path, params=None, timeout=30):
|
|
"""GET an e621 API path using the user's credentials.
|
|
|
|
Raises E621NotFound for 404s and E621Error for everything else that isn't
|
|
a 2xx, so callers never see requests exceptions.
|
|
"""
|
|
if not credentials_configured(user):
|
|
raise E621Error("Configure your e621 credentials in Account first.")
|
|
base = (getattr(user, "e621_base_url", "") or "https://e621.net").rstrip("/")
|
|
_wait_for_slot()
|
|
try:
|
|
response = requests.get(
|
|
f"{base}{path}",
|
|
params=params,
|
|
auth=(user.e621_username, user.e621_api_key),
|
|
headers={"User-Agent": settings.USER_AGENT},
|
|
timeout=timeout,
|
|
)
|
|
except requests.RequestException as exc:
|
|
raise E621Error(f"Could not reach e621: {exc}") from exc
|
|
if response.status_code == 404:
|
|
raise E621NotFound(f"e621 returned 404 for {path}")
|
|
if response.status_code >= 400:
|
|
raise E621Error(f"e621 replied {response.status_code} for {path}")
|
|
try:
|
|
return response.json()
|
|
except ValueError as exc:
|
|
raise E621Error("e621 returned an unexpected response.") from exc
|
|
|
|
|
|
def find_post_by_md5(user, md5):
|
|
"""The e621 post with this exact MD5, or None."""
|
|
payload = get(user, "/posts.json", params={"tags": f"md5:{md5}", "limit": 1})
|
|
posts = payload.get("posts") if isinstance(payload, dict) else None
|
|
if not posts:
|
|
return None
|
|
return posts[0]
|
|
|
|
|
|
def fetch_post(user, post_id):
|
|
"""One post by id. Raises E621NotFound when the post is gone."""
|
|
payload = get(user, f"/posts/{int(post_id)}.json")
|
|
post = payload.get("post") if isinstance(payload, dict) else None
|
|
if not isinstance(post, dict):
|
|
raise E621Error("e621 returned an unexpected post payload.")
|
|
return post
|