Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
041a9d4471 | ||
|
|
d84fdd0e98 | ||
|
|
c992a63b8f | ||
|
|
bd2417aff8 |
@@ -0,0 +1,108 @@
|
||||
"""Redis cache that degrades instead of taking the whole API down.
|
||||
|
||||
Redis backs the DRF throttles and a few caches (storage stats, guest
|
||||
blacklist, tag clouds). With Django's stock ``RedisCache``, a Redis that is
|
||||
unreachable — or merely refusing writes because its RDB snapshot failed, the
|
||||
default ``stop-writes-on-bgsave-error`` behaviour — raises inside the
|
||||
throttle check on every request, so a cache outage becomes a blanket 500.
|
||||
|
||||
This backend treats cache failures as misses: rate limits and cached values
|
||||
simply stop working until Redis is back, and the first failure per worker is
|
||||
logged once so the cause is still visible.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from django.core.cache.backends.redis import RedisCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_warned = False
|
||||
|
||||
|
||||
def _degrade(operation: str, error: Exception, default):
|
||||
global _warned
|
||||
if not _warned:
|
||||
_warned = True
|
||||
logger.warning(
|
||||
"Cache unavailable (%s failed: %s) — continuing without it.",
|
||||
operation,
|
||||
error,
|
||||
)
|
||||
return default
|
||||
|
||||
|
||||
class ResilientRedisCache(RedisCache):
|
||||
"""``RedisCache`` where a broken Redis behaves like an empty cache."""
|
||||
|
||||
def add(self, *args, **kwargs):
|
||||
try:
|
||||
return super().add(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001 - any backend failure degrades
|
||||
return _degrade("add", error, False)
|
||||
|
||||
def get(self, key, default=None, version=None):
|
||||
try:
|
||||
return super().get(key, default, version)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("get", error, default)
|
||||
|
||||
def set(self, *args, **kwargs):
|
||||
try:
|
||||
return super().set(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("set", error, None)
|
||||
|
||||
def touch(self, *args, **kwargs):
|
||||
try:
|
||||
return super().touch(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("touch", error, False)
|
||||
|
||||
def delete(self, *args, **kwargs):
|
||||
try:
|
||||
return super().delete(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("delete", error, False)
|
||||
|
||||
def get_many(self, *args, **kwargs):
|
||||
try:
|
||||
return super().get_many(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("get_many", error, {})
|
||||
|
||||
def has_key(self, *args, **kwargs):
|
||||
try:
|
||||
return super().has_key(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("has_key", error, False)
|
||||
|
||||
def incr(self, *args, **kwargs):
|
||||
try:
|
||||
return super().incr(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("incr", error, None)
|
||||
|
||||
def set_many(self, *args, **kwargs):
|
||||
try:
|
||||
return super().set_many(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("set_many", error, [])
|
||||
|
||||
def delete_many(self, *args, **kwargs):
|
||||
try:
|
||||
return super().delete_many(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("delete_many", error, None)
|
||||
|
||||
def clear(self, *args, **kwargs):
|
||||
try:
|
||||
return super().clear(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("clear", error, None)
|
||||
|
||||
def close(self, *args, **kwargs):
|
||||
try:
|
||||
return super().close(*args, **kwargs)
|
||||
except Exception as error: # noqa: BLE001
|
||||
return _degrade("close", error, None)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""A broken Redis must degrade the cache, not 500 the API.
|
||||
|
||||
A Redis that cannot persist (the default ``stop-writes-on-bgsave-error``)
|
||||
or is simply unreachable used to raise inside the DRF throttle check on
|
||||
every request; ``ResilientRedisCache`` treats that as a cache miss.
|
||||
"""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.core.cache.backends.redis import RedisCacheClient
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
|
||||
def failing(method: str):
|
||||
return mock.patch.object(
|
||||
RedisCacheClient,
|
||||
method,
|
||||
side_effect=RuntimeError("redis is down"),
|
||||
)
|
||||
|
||||
|
||||
class ResilientCacheTests(SimpleTestCase):
|
||||
def test_get_returns_the_default_when_redis_fails(self):
|
||||
with failing("get"):
|
||||
self.assertIsNone(cache.get("j621-cache-test"))
|
||||
self.assertEqual(cache.get("j621-cache-test", "fallback"), "fallback")
|
||||
|
||||
def test_writes_report_failure_without_raising(self):
|
||||
with failing("set"):
|
||||
self.assertIsNone(cache.set("j621-cache-test", "value"))
|
||||
|
||||
def test_bulk_and_delete_operations_degrade(self):
|
||||
with failing("get_many"), failing("delete"):
|
||||
self.assertEqual(cache.get_many(["a", "b"]), {})
|
||||
self.assertFalse(cache.delete("a"))
|
||||
@@ -242,7 +242,7 @@ SIMILARITY_TTL_MINUTES = int(os.getenv("SIMILARITY_TTL_MINUTES", "30"))
|
||||
# workers and management commands (e.g. the mirrored guest blacklist).
|
||||
CACHES = {
|
||||
"default": {
|
||||
"BACKEND": "django.core.cache.backends.redis.RedisCache",
|
||||
"BACKEND": "apps.core.cache.ResilientRedisCache",
|
||||
"LOCATION": os.getenv("REDIS_URL", "redis://127.0.0.1:6380/1"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +41,9 @@ esac
|
||||
VERSION="$(node -p "require('./desktop/package.json').version")"
|
||||
|
||||
case "$TARGET" in
|
||||
linux) ARTIFACTS=(-name "*.deb" -o -name "*.pkg.tar.zst") ;;
|
||||
win) ARTIFACTS=(-name "*.exe") ;;
|
||||
all) ARTIFACTS=(-name "*.deb" -o -name "*.pkg.tar.zst" -o -name "*.exe") ;;
|
||||
linux) ARTIFACTS=(-name "*${VERSION}*.deb" -o -name "*${VERSION}*.pkg.tar.zst") ;;
|
||||
win) ARTIFACTS=(-name "*${VERSION}*.exe") ;;
|
||||
all) ARTIFACTS=(-name "*${VERSION}*.deb" -o -name "*${VERSION}*.pkg.tar.zst" -o -name "*${VERSION}*.exe") ;;
|
||||
esac
|
||||
|
||||
echo
|
||||
|
||||
+15
-6
@@ -54,20 +54,29 @@ FEED=deploy/data/desktop
|
||||
mkdir -p "$FEED"
|
||||
|
||||
shopt -s nullglob
|
||||
deb=(desktop/release/*.deb)
|
||||
zst=(desktop/release/*.pkg.tar.zst)
|
||||
VERSION="$(node -p "require('./desktop/package.json').version")"
|
||||
deb=(desktop/release/*"$VERSION"*.deb)
|
||||
zst=(desktop/release/*"$VERSION"*.pkg.tar.zst)
|
||||
linux_meta=(desktop/release/latest-linux.yml)
|
||||
win_exe=("desktop/release/J621 Setup "*.exe)
|
||||
win_exe=("desktop/release/J621 Setup ${VERSION}"*.exe)
|
||||
win_meta=(desktop/release/latest.yml)
|
||||
blockmaps=(desktop/release/*.blockmap)
|
||||
blockmaps=(desktop/release/*"$VERSION"*.exe.blockmap)
|
||||
|
||||
if [ "${#deb[@]}" -eq 0 ] && [ "${#zst[@]}" -eq 0 ]; then
|
||||
echo "No artifacts in desktop/release/ — run without --no-build first." >&2
|
||||
echo "No $VERSION artifacts in desktop/release/ — run without --no-build first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Replace the previous release's metadata before copying the new artifacts.
|
||||
# Replace the previous release's metadata and drop older installers from the
|
||||
# feed: latest*.yml only ever points at the current version.
|
||||
rm -f "$FEED"/latest-linux.yml "$FEED"/latest.yml
|
||||
old=("$FEED"/*.deb "$FEED"/*.pkg.tar.zst "$FEED"/"J621 Setup "*.exe "$FEED"/*.exe.blockmap)
|
||||
for file in "${old[@]}"; do
|
||||
case "$(basename "$file")" in
|
||||
*"$VERSION"*) ;;
|
||||
*) echo " pruning $(basename "$file")"; rm -f "$file" ;;
|
||||
esac
|
||||
done
|
||||
cp -f "${deb[@]}" "${zst[@]}" "${linux_meta[@]}" "$FEED"/ 2>/dev/null || true
|
||||
if [ "${#win_exe[@]}" -gt 0 ]; then
|
||||
cp -f "${win_exe[@]}" "${win_meta[@]}" "${blockmaps[@]}" "$FEED"/ 2>/dev/null || true
|
||||
|
||||
@@ -20,6 +20,11 @@ src/preload.ts window.j621Desktop bridge
|
||||
electron-builder.yml deb + pacman + nsis packaging
|
||||
```
|
||||
|
||||
The main process attaches a `Referer` to requests for e621 hosts: Chromium
|
||||
sends no referrer from a custom-scheme page, and e621's CDN answers
|
||||
cross-site image loads without one with a 403 (which Chromium then blocks as
|
||||
ORB). API calls work either way.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "j621-desktop",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "j621-desktop",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.1",
|
||||
"license": "LicenseRef-Jake-Labs-Non-Commercial",
|
||||
"dependencies": {
|
||||
"electron-updater": "^6.8.9"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "j621-desktop",
|
||||
"productName": "J621",
|
||||
"version": "0.1.0",
|
||||
"version": "0.1.1",
|
||||
"private": true,
|
||||
"description": "Desktop shell for the J621 self-hosted media archive",
|
||||
"author": {
|
||||
|
||||
+67
-2
@@ -12,9 +12,9 @@
|
||||
* the same way it does in a browser. The only desktop-specific bits are the
|
||||
* origin, external-link handling and (later) updates.
|
||||
*/
|
||||
import { app, BrowserWindow, dialog, ipcMain, Menu, protocol, shell } from "electron";
|
||||
import { app, BrowserWindow, dialog, ipcMain, Menu, protocol, session, shell } from "electron";
|
||||
import { autoUpdater } from "electron-updater";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { readFileSync, rmSync } from "node:fs";
|
||||
import { readFile, stat, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -24,6 +24,15 @@ const APP_ORIGIN = `${SCHEME}://${HOST}`;
|
||||
const DEV_SERVER = (process.env.J621_DEV_SERVER ?? "").replace(/\/+$/, "");
|
||||
const SMOKE = process.argv.includes("--j621-smoke");
|
||||
|
||||
if (SMOKE) {
|
||||
// A throwaway profile keeps the checks deterministic (always the first-run
|
||||
// setup screen) and never touches real state or leaves a stale
|
||||
// single-instance lock behind.
|
||||
const smokeDir = path.join(app.getPath("temp"), "j621-smoke");
|
||||
rmSync(smokeDir, { recursive: true, force: true });
|
||||
app.setPath("userData", smokeDir);
|
||||
}
|
||||
|
||||
protocol.registerSchemesAsPrivileged([
|
||||
{
|
||||
scheme: SCHEME,
|
||||
@@ -83,6 +92,29 @@ async function fileResponse(file: string): Promise<Response> {
|
||||
});
|
||||
}
|
||||
|
||||
/** Paths the nginx proxy sends to Django; there is no backend behind app://. */
|
||||
const BACKEND_PREFIXES = ["/api/", "/admin/", "/static/", "/health"];
|
||||
|
||||
/**
|
||||
* e621's CDN refuses cross-site image loads that carry no Referer
|
||||
* (`Sec-Fetch-Site: cross-site` with an empty referrer returns 403), and
|
||||
* Chromium never sends one for pages on a custom scheme like app://j621.
|
||||
* Attach a normal e621 referrer to its hosts so images load; API calls are
|
||||
* unaffected.
|
||||
*/
|
||||
function installRefererFix(targetSession: Electron.Session): void {
|
||||
targetSession.webRequest.onBeforeSendHeaders(
|
||||
{ urls: ["*://e621.net/*", "*://*.e621.net/*"] },
|
||||
(details, callback) => {
|
||||
const headers = details.requestHeaders;
|
||||
if (!headers.Referer && !headers.referer) {
|
||||
headers.Referer = "https://e621.net/";
|
||||
}
|
||||
callback({ requestHeaders: headers });
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function handleAppRequest(request: Request): Promise<Response> {
|
||||
const url = new URL(request.url);
|
||||
if (url.host !== HOST) return new Response("Not found", { status: 404 });
|
||||
@@ -91,6 +123,21 @@ async function handleAppRequest(request: Request): Promise<Response> {
|
||||
let pathname = decodeURIComponent(url.pathname);
|
||||
if (!pathname || pathname === "/") pathname = "/index.html";
|
||||
|
||||
// Never answer backend paths with the SPA: that made the setup screen's
|
||||
// connection test "succeed" against the shell's own origin.
|
||||
if (
|
||||
BACKEND_PREFIXES.some(
|
||||
(prefix) => pathname === prefix.replace(/\/$/, "") || pathname.startsWith(prefix),
|
||||
)
|
||||
) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
detail: "No backend is attached to the desktop app; set one in /setup.",
|
||||
}),
|
||||
{ status: 404, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
}
|
||||
|
||||
const target = path.resolve(root, "." + pathname);
|
||||
if (target !== root && !target.startsWith(root + path.sep)) {
|
||||
return new Response("Forbidden", { status: 403 });
|
||||
@@ -426,6 +473,7 @@ function runSmokeTest(win: BrowserWindow): void {
|
||||
const asset = await fetch("/favicon.svg");
|
||||
const route = await fetch("/gallery/some/deep/route");
|
||||
const routeBody = await route.text();
|
||||
const health = await fetch("/health");
|
||||
localStorage.setItem("j621.smoke", "ok");
|
||||
history.pushState({}, "", "/gallery");
|
||||
return {
|
||||
@@ -433,6 +481,18 @@ function runSmokeTest(win: BrowserWindow): void {
|
||||
title: document.title,
|
||||
assetOk: asset.ok && (await asset.text()).includes("<svg"),
|
||||
routeOk: route.ok && routeBody.includes('id="root"'),
|
||||
// /health must not fall back to the SPA (false "connected" test).
|
||||
healthOk: health.status === 404,
|
||||
// Testing an empty backend URL on the desktop must say so instead
|
||||
// of "Connected" against the shell's own origin.
|
||||
emptyTestOk: await (async () => {
|
||||
const button = [...document.querySelectorAll("button")].find((b) =>
|
||||
b.textContent.includes("Test connection"),
|
||||
);
|
||||
if (!button) return false;
|
||||
button.click();
|
||||
return waitFor(() => document.body.innerText.includes("no backend of its own"));
|
||||
})(),
|
||||
mounted: await waitFor(() => (document.querySelector("#root")?.childElementCount ?? 0) > 0),
|
||||
setupOk: document.body.innerText.includes("Where is your backend?"),
|
||||
storageOk: localStorage.getItem("j621.smoke") === "ok",
|
||||
@@ -452,6 +512,10 @@ function runSmokeTest(win: BrowserWindow): void {
|
||||
routeOk: true,
|
||||
mounted: true,
|
||||
setupOk: !DEV_SERVER,
|
||||
// Both are app://-only checks: the Vite dev server serves the SPA
|
||||
// for /health and never shows the setup screen.
|
||||
healthOk: !DEV_SERVER,
|
||||
emptyTestOk: !DEV_SERVER,
|
||||
storageOk: true,
|
||||
historyOk: true,
|
||||
opfsOk: true,
|
||||
@@ -517,6 +581,7 @@ if (!gotLock) {
|
||||
|
||||
app.whenReady().then(() => {
|
||||
protocol.handle(SCHEME, handleAppRequest);
|
||||
installRefererFix(session.defaultSession);
|
||||
|
||||
ipcMain.handle("j621:version", () => app.getVersion());
|
||||
ipcMain.handle("j621:open-external", (_event, url: unknown) => {
|
||||
|
||||
@@ -17,6 +17,16 @@ import { setToken } from "@/lib/api";
|
||||
type TestResult = { ok: boolean; text: string } | null;
|
||||
|
||||
async function testBackend(base: string): Promise<TestResult> {
|
||||
// The desktop shell has no server of its own: an empty URL is not a
|
||||
// same-origin backend, and /health on app://j621 is answered by the shell.
|
||||
if (!base && window.j621Desktop) {
|
||||
return {
|
||||
ok: false,
|
||||
text:
|
||||
"The desktop app has no backend of its own. Enter your J621 server's " +
|
||||
"URL, or continue without a backend.",
|
||||
};
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timeout = window.setTimeout(() => controller.abort(), 5_000);
|
||||
try {
|
||||
@@ -110,10 +120,15 @@ export function SetupPage() {
|
||||
</div>
|
||||
|
||||
<p className="mt-3 text-sm leading-relaxed text-ctp-subtext0">
|
||||
Enter the origin this app should call for its API. Leave it blank
|
||||
when the app and the API are served from the same domain. Without a
|
||||
backend the app runs in local mode: e621 browsing only, with
|
||||
credentials kept in this browser.
|
||||
{window.j621Desktop
|
||||
? "Enter the origin of your J621 server (for example " +
|
||||
"https://j621.example.com). Without a backend the app runs in " +
|
||||
"local mode: e621 browsing only, with credentials kept on this " +
|
||||
"device."
|
||||
: "Enter the origin this app should call for its API. Leave it " +
|
||||
"blank when the app and the API are served from the same " +
|
||||
"domain. Without a backend the app runs in local mode: e621 " +
|
||||
"browsing only, with credentials kept in this browser."}
|
||||
</p>
|
||||
|
||||
<label className="mt-5 flex flex-col gap-1.5">
|
||||
@@ -122,7 +137,11 @@ export function SetupPage() {
|
||||
</span>
|
||||
<input
|
||||
className={cn(inputClass, "font-mono")}
|
||||
placeholder="https://j621.example.com — blank for this server"
|
||||
placeholder={
|
||||
window.j621Desktop
|
||||
? "https://j621.example.com"
|
||||
: "https://j621.example.com — blank for this server"
|
||||
}
|
||||
value={value}
|
||||
onChange={(event) => {
|
||||
setValue(event.target.value);
|
||||
@@ -161,7 +180,7 @@ export function SetupPage() {
|
||||
{testing ? <Spinner className="h-3.5 w-3.5" /> : null}
|
||||
{testing ? "Testing…" : "Test connection"}
|
||||
</Button>
|
||||
{value.trim() ? (
|
||||
{value.trim() && !window.j621Desktop ? (
|
||||
<Button variant="ghost" onClick={() => save("")}>
|
||||
Use this server
|
||||
</Button>
|
||||
@@ -172,8 +191,10 @@ export function SetupPage() {
|
||||
</div>
|
||||
|
||||
<p className="mt-4 text-xs leading-relaxed text-ctp-overlay0">
|
||||
Stored in this browser only. Changing the backend signs you out of
|
||||
the previous one.
|
||||
{window.j621Desktop
|
||||
? "Stored in this app only."
|
||||
: "Stored in this browser only."}{" "}
|
||||
Changing the backend signs you out of the previous one.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user