JakeBreath/J621
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Stop storing completed uploads; announce them through a live feed

Browse Source 
Every auto-matched, duplicate or manually resolved upload left a completed
TempUpload row on the board until it was dismissed by hand, so the rows
accumulated without bound and the bulk dismiss (capped at 1000 ids) failed
once there were more. The original app never stored these: they are
notifications, not records.

- complete_temp_upload now appends {filename, J-ID, resolution, post} to a
  bounded recent_completions feed on UploadRun and deletes the staged row
- staging duplicates never create a board record either; the create response
  carries the J-ID and preview so the SPA can show the card immediately
- status_payload returns the feed (newest first, signed thumbnails) for the
  live board; finalize_round counts deleted matches in processed
- resolve/link-bulk return synthetic completion payloads
- migration 0012 adds the field and purges the existing completed backlog
  (and any stray staged files) on deploy
This commit is contained in:
JakeBreath
2026-09-23 18:24:38 -05:00
parent 37085c5dac
commit 9ababb8b48
5 changed files with 237 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/library/tests/test_uploads.py
+81 -21
View File
@@ -170,12 +170,18 @@ class StagedUploadWorkflowTests(TestCase):
self.assertEqual(len(body["resolved"]), 2)
self.assertEqual(body["errors"], [])
for temp in (first, second):
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertIsNotNone(temp.library_item_id)
self.assertEqual(temp.library_item.rating, "q")
self.assertEqual(temp.library_item.uploaded_by_id, self.uploader.id)
# Completed uploads are notifications now: the staged rows are gone
# and the live feed carries the filename -> J-ID mapping.
self.assertFalse(
TempUpload.objects.filter(pk__in=[first.id, second.id]).exists()
)
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(
{entry["filename"] for entry in feed}, {"one.png", "two.png"}
)
for item in MediaItem.objects.all():
self.assertEqual(item.rating, "q")
self.assertEqual(item.uploaded_by_id, self.uploader.id)
untouched.refresh_from_db()
self.assertEqual(untouched.status, TempUpload.STATUS_PENDING)
@@ -231,6 +237,25 @@ class StagedUploadWorkflowTests(TestCase):
self.assertEqual(response.status_code, 200)
return seed
def test_duplicate_upload_returns_a_completion_without_a_record(self):
client = self.api_client(self.uploader)
first = self.upload_via_api(client, "same.png")
self.assertEqual(first.status_code, 201)
# Index the first upload so the second one is byte-identical.
seed = TempUpload.objects.get(pk=first.json()["temp_id"])
self.resolve_bulk(client, [seed.id], "s")
response = self.upload_via_api(client, "same.png")
self.assertEqual(response.status_code, 201)
body = response.json()
self.assertEqual(body["status"], TempUpload.STATUS_COMPLETED)
self.assertEqual(body["resolution"], TempUpload.RESOLUTION_DUPLICATE)
self.assertTrue(body["library_j_id"].startswith("J-"))
# Duplicates never become board records; the feed announces them.
self.assertFalse(TempUpload.objects.filter(pk=body["temp_id"]).exists())
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(feed[-1]["filename"], "same.png")
def test_upload_defers_visual_similarity_to_its_phase(self):
client = self.api_client(self.uploader)
self.seed_library_item(client, "seed-defer")
@@ -312,14 +337,23 @@ class StagedUploadWorkflowTests(TestCase):
body = response.json()
self.assertEqual(body["errors"], [])
self.assertEqual(len(body["updated"]), 2)
for temp, post_id in ((first, 900001), (second, 900002)):
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertEqual(temp.library_item_id is not None, True)
self.assertEqual(temp.e621_post_id, post_id)
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(temp.library_item.e621_post_id, post_id)
self.assertEqual(MediaItem.objects.count(), 2)
for entry, post_id in zip(body["updated"], (900001, 900002)):
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(entry["e621_post_id"], post_id)
self.assertTrue(entry["library_j_id"].startswith("J-"))
# Indexed uploads no longer leave a board record; the completion feed
# carries them for the live page instead.
self.assertFalse(TempUpload.objects.exists())
self.assertEqual(
{item.e621_post_id for item in MediaItem.objects.all()},
{900001, 900002},
)
feed = UploadRun.objects.get(user=self.uploader).recent_completions
self.assertEqual(len(feed), 2)
self.assertEqual(
{entry["filename"] for entry in feed},
{"bulk-link-1.png", "bulk-link-2.png"},
)
class IqdbRecordingTests(TestCase):
@@ -461,6 +495,7 @@ class UploadPipelineTests(TestCase):
def test_md5_match_auto_imports_the_file(self):
temp = self.stage(label="match")
temp_id = temp.id
post = {
"id": 123456,
"rating": "s",
@@ -477,17 +512,42 @@ class UploadPipelineTests(TestCase):
):
upload_pipeline.run_pipeline(self.uploader.id)
temp.refresh_from_db()
self.assertEqual(temp.status, TempUpload.STATUS_COMPLETED)
self.assertEqual(temp.resolution, TempUpload.RESOLUTION_AUTO_MD5)
self.assertEqual(temp.e621_post_id, 123456)
self.assertIsNotNone(temp.library_item_id)
self.assertIsNotNone(temp.e621_checked_at)
self.assertIsNone(temp.claimed_at)
# The indexed upload leaves no board record; the feed reports it.
self.assertFalse(TempUpload.objects.filter(pk=temp_id).exists())
item = MediaItem.objects.get(e621_post_id=123456)
self.assertEqual(item.uploaded_by_id, self.uploader.id)
run = UploadRun.objects.get(user=self.uploader)
self.assertEqual(run.status, UploadRun.STATUS_IDLE)
self.assertEqual(run.matched, 1)
self.assertEqual(run.processed, 1)
self.assertEqual(len(run.recent_completions), 1)
entry = run.recent_completions[0]
self.assertEqual(entry["id"], str(temp_id))
self.assertEqual(entry["item_id"], item.id)
self.assertEqual(entry["filename"], "match.png")
self.assertEqual(entry["resolution"], TempUpload.RESOLUTION_AUTO_MD5)
def test_status_reports_the_completion_feed(self):
temp = self.stage(label="feed")
client = self.api_client(self.uploader)
post = {
"id": 654321,
"rating": "s",
"file": {"md5": temp.md5, "url": "https://static1.e621.net/data/f.png"},
}
with mock.patch.object(
upload_pipeline.e621,
"check_md5_batch",
return_value={temp.md5: post},
):
upload_pipeline.run_pipeline(self.uploader.id)
body = client.get("/api/uploads/status/").json()
completions = body["recent_completions"]
self.assertEqual(len(completions), 1)
self.assertEqual(completions[0]["filename"], "feed.png")
self.assertEqual(completions[0]["j_id"], f"J-{MediaItem.objects.get().id}")
self.assertIn("/thumbnail/", completions[0]["thumbnail_url"])
def test_unmatched_file_runs_every_phase(self):
temp = self.stage(label="nomatch")