JakeBreath
9641862515
e621 intermittently answers 429 to the IQDB endpoint; browsers hide that
status behind CORS ('Access-Control-Allow-Origin missing'), so the SPA
cannot read it. Treat every network-level failure as a possible rate limit
and pause all e621 traffic for a minute. The cooldown is shared through
localStorage so extra tabs respect it, requests are spaced 1.5s apart
instead of 1s, user-cancelled requests do not trigger a cooldown, and the
upload queue waits the cooldown out with a countdown instead of looking
stuck.
Server side: the per-process e621 gap goes from 0.5s to 1s so two gunicorn
workers cannot together exceed e621's 2/s hard limit.
101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
"""Minimal e621 API client for server-side matching and metadata refresh.
|
|
|
|
The SPA talks to e621 directly for browsing; this client exists for work the
|
|
browser cannot do reliably: long batch scans, and requests tied to a library
|
|
item rather than an open page. It uses the requesting user's stored
|
|
credentials and a global throttle (e621 asks for at most two requests per
|
|
second).
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
|
|
# Seconds between requests, per process. e621 allows 2/s hard and 1/s
|
|
# sustained; each gunicorn worker throttles on its own, so leave enough
|
|
# headroom that combined traffic does not trip the limit.
|
|
REQUEST_INTERVAL = 1.0
|
|
|
|
|
|
class E621Error(Exception):
|
|
"""A non-404 failure while talking to e621."""
|
|
|
|
|
|
class E621NotFound(E621Error):
|
|
"""The requested post does not exist (HTTP 404)."""
|
|
|
|
|
|
_throttle_lock = threading.Lock()
|
|
_last_request_at = 0.0
|
|
|
|
|
|
def credentials_configured(user):
|
|
return bool(user is not None and getattr(user, "e621_configured", False))
|
|
|
|
|
|
def _wait_for_slot():
|
|
global _last_request_at
|
|
with _throttle_lock:
|
|
delay = _last_request_at + REQUEST_INTERVAL - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
_last_request_at = time.monotonic()
|
|
|
|
|
|
def get(user, path, params=None, timeout=30, require_auth=True):
|
|
"""GET an e621 API path using the user's credentials.
|
|
|
|
Reads that e621 serves anonymously (searches, pools, tags) can pass
|
|
require_auth=False; matching endpoints keep requiring credentials.
|
|
|
|
Raises E621NotFound for 404s and E621Error for everything else that isn't
|
|
a 2xx, so callers never see requests exceptions.
|
|
"""
|
|
configured = credentials_configured(user)
|
|
if require_auth and not configured:
|
|
raise E621Error("Configure your e621 credentials in Account first.")
|
|
base = (getattr(user, "e621_base_url", "") or "https://e621.net").rstrip("/")
|
|
_wait_for_slot()
|
|
try:
|
|
response = requests.get(
|
|
f"{base}{path}",
|
|
params=params,
|
|
auth=(
|
|
(user.e621_username, user.e621_api_key_plain)
|
|
if configured
|
|
else None
|
|
),
|
|
headers={"User-Agent": settings.USER_AGENT},
|
|
timeout=timeout,
|
|
)
|
|
except requests.RequestException as exc:
|
|
raise E621Error(f"Could not reach e621: {exc}") from exc
|
|
if response.status_code == 404:
|
|
raise E621NotFound(f"e621 returned 404 for {path}")
|
|
if response.status_code >= 400:
|
|
raise E621Error(f"e621 replied {response.status_code} for {path}")
|
|
try:
|
|
return response.json()
|
|
except ValueError as exc:
|
|
raise E621Error("e621 returned an unexpected response.") from exc
|
|
|
|
|
|
def find_post_by_md5(user, md5):
|
|
"""The e621 post with this exact MD5, or None."""
|
|
payload = get(user, "/posts.json", params={"tags": f"md5:{md5}", "limit": 1})
|
|
posts = payload.get("posts") if isinstance(payload, dict) else None
|
|
if not posts:
|
|
return None
|
|
return posts[0]
|
|
|
|
|
|
def fetch_post(user, post_id):
|
|
"""One post by id. Raises E621NotFound when the post is gone."""
|
|
payload = get(user, f"/posts/{int(post_id)}.json")
|
|
post = payload.get("post") if isinstance(payload, dict) else None
|
|
if not isinstance(post, dict):
|
|
raise E621Error("e621 returned an unexpected post payload.")
|
|
return post
|