JakeBreath/J621
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Stop storing completed uploads; announce them through a live feed

Browse Source 
Every auto-matched, duplicate or manually resolved upload left a completed
TempUpload row on the board until it was dismissed by hand, so the rows
accumulated without bound and the bulk dismiss (capped at 1000 ids) failed
once there were more. The original app never stored these: they are
notifications, not records.

- complete_temp_upload now appends {filename, J-ID, resolution, post} to a
  bounded recent_completions feed on UploadRun and deletes the staged row
- staging duplicates never create a board record either; the create response
  carries the J-ID and preview so the SPA can show the card immediately
- status_payload returns the feed (newest first, signed thumbnails) for the
  live board; finalize_round counts deleted matches in processed
- resolve/link-bulk return synthetic completion payloads
- migration 0012 adds the field and purges the existing completed backlog
  (and any stray staged files) on deploy
This commit is contained in:
JakeBreath
2026-09-23 18:24:38 -05:00
parent 37085c5dac
commit 9ababb8b48
5 changed files with 237 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/library/uploads.py
+44 -10
View File
@@ -158,6 +158,13 @@ def complete_temp_upload(temp, download_url=None):
item.save(update_fields=update_fields + ["updated_at"])
temp.save()
from .upload_pipeline import record_completion
record_completion(temp, item)
# The board learns about completions from the live feed, so the record is
# deleted as soon as the file is indexed: nothing left to dismiss. Delete
# through the queryset so callers keep ``temp.pk`` for their response.
TempUpload.objects.filter(pk=temp.pk).delete()
return item
@@ -190,6 +197,28 @@ class TempUploadViewSet(
user=self.request.user
)
def _completed_payload(self, temp, item):
"""Synthetic row for an upload that is indexed immediately.
Duplicates and resolved uploads never leave a board record; the SPA
turns this response (or the live completion feed) into a "J-x
uploaded" card that lives only in the page session.
"""
return {
"temp_id": str(temp.pk),
"original_filename": temp.original_filename,
"md5": temp.md5,
"size": temp.size,
"status": TempUpload.STATUS_COMPLETED,
"resolution": temp.resolution,
"e621_post_id": temp.e621_post_id,
"library_j_id": f"J-{item.id}",
"file_url": None,
"preview_url": services.signed_media_url(
item, self.request.user, "thumbnail", request=self.request
),
}
def create(self, request):
upload = request.FILES.get("file")
if upload is None:
@@ -217,6 +246,13 @@ class TempUploadViewSet(
temp.resolution = TempUpload.RESOLUTION_DUPLICATE
temp.library_item = existing
temp.file.delete(save=False)
temp.save()
from .upload_pipeline import record_completion
record_completion(temp, existing)
payload = self._completed_payload(temp, existing)
TempUpload.objects.filter(pk=temp.pk).delete()
return Response(payload, status=status.HTTP_201_CREATED)
# Visual similarity and IQDB run in the background pipeline so a large
# batch uploads at full speed and the work survives the browser.
temp.save()
@@ -257,7 +293,7 @@ class TempUploadViewSet(
"""Cheap pipeline state for the shell indicator and the upload page."""
from .upload_pipeline import status_payload
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"])
def process(self, request):
@@ -269,7 +305,7 @@ class TempUploadViewSet(
from .upload_pipeline import start_pipeline, status_payload
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=True, methods=["post"])
def retry(self, request, pk=None):
@@ -314,7 +350,7 @@ class TempUploadViewSet(
update["status"] = TempUpload.STATUS_PENDING
TempUpload.objects.filter(pk=temp.pk).update(**update)
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"], url_path="retry-all")
def retry_all(self, request):
@@ -340,7 +376,7 @@ class TempUploadViewSet(
e621_checked_at=None,
)
start_pipeline(request.user)
return Response(status_payload(request.user))
return Response(status_payload(request.user, request=request))
@action(detail=False, methods=["post"], url_path="discard-bulk")
def discard_bulk(self, request):
@@ -494,7 +530,7 @@ class TempUploadViewSet(
temp.save()
try:
complete_temp_upload(temp, download_url=download_url)
item = complete_temp_upload(temp, download_url=download_url)
except Exception as exc: # noqa: BLE001 - report completion failures
logger.exception("Could not complete staged upload %s", temp.id)
temp.status = TempUpload.STATUS_ERROR
@@ -503,8 +539,7 @@ class TempUploadViewSet(
{"detail": f"Could not finish the upload: {exc}"},
status=status.HTTP_400_BAD_REQUEST,
)
temp.refresh_from_db()
return Response(self.get_serializer(temp).data)
return Response(self._completed_payload(temp, item))
@action(detail=False, methods=["post"], url_path="link-bulk")
def link_bulk(self, request):
@@ -576,15 +611,14 @@ class TempUploadViewSet(
candidate_url = ""
temp.save()
try:
complete_temp_upload(temp, download_url=candidate_url or None)
item = complete_temp_upload(temp, download_url=candidate_url or None)
except Exception as exc: # noqa: BLE001 - report per-file failures
logger.exception("Could not complete staged upload %s", temp.id)
temp.status = TempUpload.STATUS_ERROR
temp.save(update_fields=["status", "updated_at"])
errors.append({"temp_id": temp_id, "error": str(exc)})
continue
temp.refresh_from_db()
updated.append(self.get_serializer(temp).data)
updated.append(self._completed_payload(temp, item))
return Response({"updated": updated, "errors": errors})