JakeBreath
bd2417aff8
Redis backs the DRF throttles, and the stock RedisCache raises inside the throttle check when Redis is unreachable or refusing writes (a failed RDB snapshot disables writes by default) — turning a cache problem into a blanket 500, which is exactly what took prod down. ResilientRedisCache treats backend failures as cache misses, logs the first one per worker, and lets rate limits degrade until Redis is back.
109 lines
3.6 KiB
Python
109 lines
3.6 KiB
Python
"""Redis cache that degrades instead of taking the whole API down.
|
|
|
|
Redis backs the DRF throttles and a few caches (storage stats, guest
|
|
blacklist, tag clouds). With Django's stock ``RedisCache``, a Redis that is
|
|
unreachable — or merely refusing writes because its RDB snapshot failed, the
|
|
default ``stop-writes-on-bgsave-error`` behaviour — raises inside the
|
|
throttle check on every request, so a cache outage becomes a blanket 500.
|
|
|
|
This backend treats cache failures as misses: rate limits and cached values
|
|
simply stop working until Redis is back, and the first failure per worker is
|
|
logged once so the cause is still visible.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from django.core.cache.backends.redis import RedisCache
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_warned = False
|
|
|
|
|
|
def _degrade(operation: str, error: Exception, default):
|
|
global _warned
|
|
if not _warned:
|
|
_warned = True
|
|
logger.warning(
|
|
"Cache unavailable (%s failed: %s) — continuing without it.",
|
|
operation,
|
|
error,
|
|
)
|
|
return default
|
|
|
|
|
|
class ResilientRedisCache(RedisCache):
|
|
"""``RedisCache`` where a broken Redis behaves like an empty cache."""
|
|
|
|
def add(self, *args, **kwargs):
|
|
try:
|
|
return super().add(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001 - any backend failure degrades
|
|
return _degrade("add", error, False)
|
|
|
|
def get(self, key, default=None, version=None):
|
|
try:
|
|
return super().get(key, default, version)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("get", error, default)
|
|
|
|
def set(self, *args, **kwargs):
|
|
try:
|
|
return super().set(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("set", error, None)
|
|
|
|
def touch(self, *args, **kwargs):
|
|
try:
|
|
return super().touch(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("touch", error, False)
|
|
|
|
def delete(self, *args, **kwargs):
|
|
try:
|
|
return super().delete(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("delete", error, False)
|
|
|
|
def get_many(self, *args, **kwargs):
|
|
try:
|
|
return super().get_many(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("get_many", error, {})
|
|
|
|
def has_key(self, *args, **kwargs):
|
|
try:
|
|
return super().has_key(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("has_key", error, False)
|
|
|
|
def incr(self, *args, **kwargs):
|
|
try:
|
|
return super().incr(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("incr", error, None)
|
|
|
|
def set_many(self, *args, **kwargs):
|
|
try:
|
|
return super().set_many(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("set_many", error, [])
|
|
|
|
def delete_many(self, *args, **kwargs):
|
|
try:
|
|
return super().delete_many(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("delete_many", error, None)
|
|
|
|
def clear(self, *args, **kwargs):
|
|
try:
|
|
return super().clear(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("clear", error, None)
|
|
|
|
def close(self, *args, **kwargs):
|
|
try:
|
|
return super().close(*args, **kwargs)
|
|
except Exception as error: # noqa: BLE001
|
|
return _degrade("close", error, None)
|