Match local files to e621: MD5 lookups, manual links, batch scans
- MediaItem gains e621_match_status (unknown/matched/not_found/deleted) and e621_checked_at, backfilled for existing matched items. - Server-side e621 client (apps/library/e621.py) using the user's stored credentials, throttled to 2 req/s, with typed errors. - Matching service: MD5 lookup, manual post linking (flags MD5 mismatches), unlink, metadata refresh, deleted-post detection. - Detail actions POST /api/files/J-x/match/ and /unlink/ (uploader or staff only). - Background library scans: MatchTask + /api/matches/ with missing/all scopes, progress polling, cancel and stale-task reaping; the scan counts toward the footer's Active Workers. Same pass available as manage.py match_e621 for cron. - Library gains not_found/deleted status filters; the detail page adds an e621 match card (check / link by post ID / unlink) and the metadata card warns when a post was deleted on e621.
This commit is contained in:
@@ -27,6 +27,17 @@ def build_search_tags(tags, e621_data):
|
||||
class MediaItem(models.Model):
|
||||
"""A logical media file, identified by its MD5 fingerprint."""
|
||||
|
||||
E621_UNKNOWN = "unknown"
|
||||
E621_MATCHED = "matched"
|
||||
E621_NOT_FOUND = "not_found"
|
||||
E621_DELETED = "deleted"
|
||||
E621_STATUS_CHOICES = [
|
||||
(E621_UNKNOWN, "Not checked"),
|
||||
(E621_MATCHED, "Matched"),
|
||||
(E621_NOT_FOUND, "Not found on e621"),
|
||||
(E621_DELETED, "Post deleted on e621"),
|
||||
]
|
||||
|
||||
md5 = models.CharField(max_length=32, unique=True)
|
||||
size = models.BigIntegerField(default=0)
|
||||
# Custom rating: "s" (safe), "q" (questionable), "e" (explicit) or "" (unrated).
|
||||
@@ -44,6 +55,14 @@ class MediaItem(models.Model):
|
||||
hidden_from_guests = models.BooleanField(default=False, db_index=True)
|
||||
e621_post_id = models.IntegerField(null=True, blank=True, db_index=True)
|
||||
e621_data = models.JSONField(null=True, blank=True)
|
||||
# Cached result of the last MD5/manual match against e621.
|
||||
e621_match_status = models.CharField(
|
||||
max_length=20,
|
||||
choices=E621_STATUS_CHOICES,
|
||||
default=E621_UNKNOWN,
|
||||
db_index=True,
|
||||
)
|
||||
e621_checked_at = models.DateTimeField(null=True, blank=True)
|
||||
# Perceptual hashes (hex strings) used by the duplicates engine.
|
||||
ahash = models.CharField(max_length=32, blank=True, default="", db_index=True)
|
||||
dhash = models.CharField(max_length=32, blank=True, default="", db_index=True)
|
||||
@@ -207,3 +226,58 @@ class DownloadTask(models.Model):
|
||||
|
||||
def __str__(self):
|
||||
return f"Download {self.id} ({self.status})"
|
||||
|
||||
|
||||
class MatchTask(models.Model):
|
||||
"""A background library-wide e621 match scan with progress tracking."""
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_COMPLETE = "complete"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CANCELLED = "cancelled"
|
||||
STATUS_CHOICES = [
|
||||
(STATUS_PENDING, "Pending"),
|
||||
(STATUS_RUNNING, "Running"),
|
||||
(STATUS_COMPLETE, "Complete"),
|
||||
(STATUS_ERROR, "Error"),
|
||||
(STATUS_CANCELLED, "Cancelled"),
|
||||
]
|
||||
|
||||
SCOPE_MISSING = "missing"
|
||||
SCOPE_ALL = "all"
|
||||
SCOPE_CHOICES = [
|
||||
(SCOPE_MISSING, "Unchecked items"),
|
||||
(SCOPE_ALL, "Whole library"),
|
||||
]
|
||||
|
||||
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL,
|
||||
null=True,
|
||||
blank=True,
|
||||
on_delete=models.SET_NULL,
|
||||
related_name="match_tasks",
|
||||
)
|
||||
scope = models.CharField(
|
||||
max_length=20, choices=SCOPE_CHOICES, default=SCOPE_MISSING
|
||||
)
|
||||
status = models.CharField(
|
||||
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||
)
|
||||
total = models.IntegerField(default=0)
|
||||
processed = models.IntegerField(default=0)
|
||||
matched = models.IntegerField(default=0)
|
||||
not_found = models.IntegerField(default=0)
|
||||
deleted = models.IntegerField(default=0)
|
||||
failed = models.IntegerField(default=0)
|
||||
error = models.TextField(blank=True, default="")
|
||||
cancelled = models.BooleanField(default=False)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ["-created_at"]
|
||||
|
||||
def __str__(self):
|
||||
return f"Match scan {self.id} ({self.status})"
|
||||
|
||||
Reference in New Issue
Block a user