diff --git a/backend/apps/core/cache.py b/backend/apps/core/cache.py new file mode 100644 index 0000000..f46d554 --- /dev/null +++ b/backend/apps/core/cache.py @@ -0,0 +1,108 @@ +"""Redis cache that degrades instead of taking the whole API down. + +Redis backs the DRF throttles and a few caches (storage stats, guest +blacklist, tag clouds). With Django's stock ``RedisCache``, a Redis that is +unreachable — or merely refusing writes because its RDB snapshot failed, the +default ``stop-writes-on-bgsave-error`` behaviour — raises inside the +throttle check on every request, so a cache outage becomes a blanket 500. + +This backend treats cache failures as misses: rate limits and cached values +simply stop working until Redis is back, and the first failure per worker is +logged once so the cause is still visible. +""" + +import logging + +from django.core.cache.backends.redis import RedisCache + +logger = logging.getLogger(__name__) + +_warned = False + + +def _degrade(operation: str, error: Exception, default): + global _warned + if not _warned: + _warned = True + logger.warning( + "Cache unavailable (%s failed: %s) — continuing without it.", + operation, + error, + ) + return default + + +class ResilientRedisCache(RedisCache): + """``RedisCache`` where a broken Redis behaves like an empty cache.""" + + def add(self, *args, **kwargs): + try: + return super().add(*args, **kwargs) + except Exception as error: # noqa: BLE001 - any backend failure degrades + return _degrade("add", error, False) + + def get(self, key, default=None, version=None): + try: + return super().get(key, default, version) + except Exception as error: # noqa: BLE001 + return _degrade("get", error, default) + + def set(self, *args, **kwargs): + try: + return super().set(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("set", error, None) + + def touch(self, *args, **kwargs): + try: + return super().touch(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("touch", error, False) + + def delete(self, *args, **kwargs): + try: + return super().delete(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("delete", error, False) + + def get_many(self, *args, **kwargs): + try: + return super().get_many(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("get_many", error, {}) + + def has_key(self, *args, **kwargs): + try: + return super().has_key(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("has_key", error, False) + + def incr(self, *args, **kwargs): + try: + return super().incr(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("incr", error, None) + + def set_many(self, *args, **kwargs): + try: + return super().set_many(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("set_many", error, []) + + def delete_many(self, *args, **kwargs): + try: + return super().delete_many(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("delete_many", error, None) + + def clear(self, *args, **kwargs): + try: + return super().clear(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("clear", error, None) + + def close(self, *args, **kwargs): + try: + return super().close(*args, **kwargs) + except Exception as error: # noqa: BLE001 + return _degrade("close", error, None) diff --git a/backend/apps/core/tests/test_cache.py b/backend/apps/core/tests/test_cache.py new file mode 100644 index 0000000..04f466f --- /dev/null +++ b/backend/apps/core/tests/test_cache.py @@ -0,0 +1,36 @@ +"""A broken Redis must degrade the cache, not 500 the API. + +A Redis that cannot persist (the default ``stop-writes-on-bgsave-error``) +or is simply unreachable used to raise inside the DRF throttle check on +every request; ``ResilientRedisCache`` treats that as a cache miss. +""" + +from unittest import mock + +from django.core.cache import cache +from django.core.cache.backends.redis import RedisCacheClient +from django.test import SimpleTestCase + + +def failing(method: str): + return mock.patch.object( + RedisCacheClient, + method, + side_effect=RuntimeError("redis is down"), + ) + + +class ResilientCacheTests(SimpleTestCase): + def test_get_returns_the_default_when_redis_fails(self): + with failing("get"): + self.assertIsNone(cache.get("j621-cache-test")) + self.assertEqual(cache.get("j621-cache-test", "fallback"), "fallback") + + def test_writes_report_failure_without_raising(self): + with failing("set"): + self.assertIsNone(cache.set("j621-cache-test", "value")) + + def test_bulk_and_delete_operations_degrade(self): + with failing("get_many"), failing("delete"): + self.assertEqual(cache.get_many(["a", "b"]), {}) + self.assertFalse(cache.delete("a")) diff --git a/backend/config/settings.py b/backend/config/settings.py index 488ab74..3e4d74b 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -242,7 +242,7 @@ SIMILARITY_TTL_MINUTES = int(os.getenv("SIMILARITY_TTL_MINUTES", "30")) # workers and management commands (e.g. the mirrored guest blacklist). CACHES = { "default": { - "BACKEND": "django.core.cache.backends.redis.RedisCache", + "BACKEND": "apps.core.cache.ResilientRedisCache", "LOCATION": os.getenv("REDIS_URL", "redis://127.0.0.1:6380/1"), } }