Keep a broken Redis from 500ing the whole API

Redis backs the DRF throttles, and the stock RedisCache raises inside the
throttle check when Redis is unreachable or refusing writes (a failed RDB
snapshot disables writes by default) — turning a cache problem into a
blanket 500, which is exactly what took prod down. ResilientRedisCache
treats backend failures as cache misses, logs the first one per worker, and
lets rate limits degrade until Redis is back.

backend/apps/core/cache.py

@@ -0,0 +1,108 @@
+"""Redis cache that degrades instead of taking the whole API down.
+
+Redis backs the DRF throttles and a few caches (storage stats, guest
+blacklist, tag clouds). With Django's stock ``RedisCache``, a Redis that is
+unreachable — or merely refusing writes because its RDB snapshot failed, the
+default ``stop-writes-on-bgsave-error`` behaviour — raises inside the
+throttle check on every request, so a cache outage becomes a blanket 500.
+
+This backend treats cache failures as misses: rate limits and cached values
+simply stop working until Redis is back, and the first failure per worker is
+logged once so the cause is still visible.
+"""
+
+import logging
+
+from django.core.cache.backends.redis import RedisCache
+
+logger = logging.getLogger(__name__)
+
+_warned = False
+
+
+def _degrade(operation: str, error: Exception, default):
+    global _warned
+    if not _warned:
+        _warned = True
+        logger.warning(
+            "Cache unavailable (%s failed: %s) — continuing without it.",
+            operation,
+            error,
+        )
+    return default
+
+
+class ResilientRedisCache(RedisCache):
+    """``RedisCache`` where a broken Redis behaves like an empty cache."""
+
+    def add(self, *args, **kwargs):
+        try:
+            return super().add(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001 - any backend failure degrades
+            return _degrade("add", error, False)
+
+    def get(self, key, default=None, version=None):
+        try:
+            return super().get(key, default, version)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("get", error, default)
+
+    def set(self, *args, **kwargs):
+        try:
+            return super().set(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("set", error, None)
+
+    def touch(self, *args, **kwargs):
+        try:
+            return super().touch(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("touch", error, False)
+
+    def delete(self, *args, **kwargs):
+        try:
+            return super().delete(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("delete", error, False)
+
+    def get_many(self, *args, **kwargs):
+        try:
+            return super().get_many(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("get_many", error, {})
+
+    def has_key(self, *args, **kwargs):
+        try:
+            return super().has_key(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("has_key", error, False)
+
+    def incr(self, *args, **kwargs):
+        try:
+            return super().incr(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("incr", error, None)
+
+    def set_many(self, *args, **kwargs):
+        try:
+            return super().set_many(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("set_many", error, [])
+
+    def delete_many(self, *args, **kwargs):
+        try:
+            return super().delete_many(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("delete_many", error, None)
+
+    def clear(self, *args, **kwargs):
+        try:
+            return super().clear(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("clear", error, None)
+
+    def close(self, *args, **kwargs):
+        try:
+            return super().close(*args, **kwargs)
+        except Exception as error:  # noqa: BLE001
+            return _degrade("close", error, None)

backend/apps/core/tests/test_cache.py

@@ -0,0 +1,36 @@
+"""A broken Redis must degrade the cache, not 500 the API.
+
+A Redis that cannot persist (the default ``stop-writes-on-bgsave-error``)
+or is simply unreachable used to raise inside the DRF throttle check on
+every request; ``ResilientRedisCache`` treats that as a cache miss.
+"""
+
+from unittest import mock
+
+from django.core.cache import cache
+from django.core.cache.backends.redis import RedisCacheClient
+from django.test import SimpleTestCase
+
+
+def failing(method: str):
+    return mock.patch.object(
+        RedisCacheClient,
+        method,
+        side_effect=RuntimeError("redis is down"),
+    )
+
+
+class ResilientCacheTests(SimpleTestCase):
+    def test_get_returns_the_default_when_redis_fails(self):
+        with failing("get"):
+            self.assertIsNone(cache.get("j621-cache-test"))
+            self.assertEqual(cache.get("j621-cache-test", "fallback"), "fallback")
+
+    def test_writes_report_failure_without_raising(self):
+        with failing("set"):
+            self.assertIsNone(cache.set("j621-cache-test", "value"))
+
+    def test_bulk_and_delete_operations_degrade(self):
+        with failing("get_many"), failing("delete"):
+            self.assertEqual(cache.get_many(["a", "b"]), {})
+            self.assertFalse(cache.delete("a"))

backend/config/settings.py

@@ -242,7 +242,7 @@ SIMILARITY_TTL_MINUTES = int(os.getenv("SIMILARITY_TTL_MINUTES", "30"))
 # workers and management commands (e.g. the mirrored guest blacklist).
 CACHES = {
     "default": {
-        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "BACKEND": "apps.core.cache.ResilientRedisCache",
         "LOCATION": os.getenv("REDIS_URL", "redis://127.0.0.1:6380/1"),
     }
 }