Schedule stale gallery enrichment

mobilede_scraper/worker/tasks.py

@@ -7,7 +7,7 @@ import signal
 import hashlib
 import re
 import unicodedata
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from threading import Event, Thread
 import time
 import uuid
@@ -2431,7 +2431,7 @@ def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars
 
 @shared_task(
     name="mobilede.enrich_images_batch",
-    queue=MOBILEDE_SYNC_QUEUE,
+    queue=MOBILEDE_IMAGES_QUEUE,
     bind=True,
     max_retries=1,
     default_retry_delay=120,
@@ -2439,53 +2439,100 @@ def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars
 )
 def mobilede_enrich_images_batch_task(
     self,
-    limit: int = 50,
     lane: str = "mobile_de_cars",
-    max_existing_images: int = 1,
-    delay_seconds: float = 1.0,
+    batch_size: int | None = None,
+    max_existing_images: int | None = None,
+    staleness_hours: int | None = None,
+    delay_seconds: float | None = None,
 ):
+    runtime_config = RuntimeConfig.from_file(Settings().runtime_config_file)
+    enrichment = runtime_config.mobilede.enrichment
+    if not enrichment.enabled:
+        return {
+            "status": "disabled",
+            "candidates": 0,
+            "enriched": 0,
+            "failed": 0,
+            "skipped": 0,
+        }
+
+    effective_batch_size = enrichment.batch_size if batch_size is None else max(1, int(batch_size))
+    effective_max_images = (
+        enrichment.max_existing_images
+        if max_existing_images is None
+        else max(0, int(max_existing_images))
+    )
+    effective_staleness_hours = (
+        enrichment.staleness_hours
+        if staleness_hours is None
+        else max(0, int(staleness_hours))
+    )
+    effective_delay = enrichment.delay_seconds if delay_seconds is None else max(0.0, float(delay_seconds))
+
+    redis_client = _get_redis()
+    owner_token = str(getattr(self.request, "id", None) or uuid.uuid4().hex)
+    lock_ttl = max(300, int(float(os.getenv("MOBILEDE_ENRICH_LOCK_TTL_SECONDS", "3600"))))
+    if not _acquire_lock(redis_client, MOBILEDE_ENRICH_IMAGES_LOCK_KEY, owner_token, lock_ttl):
+        return {
+            "status": "locked",
+            "candidates": 0,
+            "enriched": 0,
+            "failed": 0,
+            "skipped": 0,
+        }
+
     persistence = _get_persistence()
     scraper = MobileDeScraper(persistence=persistence)
-    candidates = persistence.get_active_cars_batch_for_image_enrich(
-        limit=limit,
-        max_existing_images=max_existing_images,
-    )
-    enriched = 0
-    failed = 0
-    skipped = 0
-    for _car_id, origin_id, origin_url, image_count in candidates:
-        listing_id = _mobilede_extract_listing_id(origin_url) or str(origin_id).rsplit(":", 1)[-1]
-        if not listing_id:
-            skipped += 1
-            continue
-        try:
-            result = scraper.sync_detail(str(listing_id), lane=lane)
-            enriched += 1
-            logger.info(
-                "mobile.de image enrich completed: listing_id=%s origin_id=%s old_images=%s result=%s",
-                listing_id,
-                origin_id,
-                image_count,
-                result.get("upsert", {}),
-            )
-        except Exception as exc:
-            failed += 1
-            logger.warning(
-                "mobile.de image enrich failed: listing_id=%s origin_id=%s error=%s",
-                listing_id,
-                origin_id,
-                exc,
-                exc_info=True,
-            )
-        if delay_seconds:
-            time.sleep(max(0.0, float(delay_seconds)))
-    return {
-        "status": "success",
-        "candidates": len(candidates),
-        "enriched": enriched,
-        "failed": failed,
-        "skipped": skipped,
-    }
+    try:
+        stale_before = datetime.now(timezone.utc) - timedelta(hours=effective_staleness_hours)
+        candidates = persistence.get_active_cars_batch_for_image_enrich(
+            max_existing_images=effective_max_images,
+            stale_before=stale_before,
+        )
+        enriched = 0
+        failed = 0
+        skipped = 0
+        for batch_start in range(0, len(candidates), effective_batch_size):
+            batch = candidates[batch_start:batch_start + effective_batch_size]
+            for _car_id, origin_id, origin_url, image_count in batch:
+                listing_id = _mobilede_extract_listing_id(origin_url) or str(origin_id).rsplit(":", 1)[-1]
+                if not listing_id:
+                    skipped += 1
+                    continue
+                try:
+                    result = scraper.sync_detail(str(listing_id), lane=lane)
+                    enriched += 1
+                    logger.info(
+                        "mobile.de image enrich completed: listing_id=%s origin_id=%s old_images=%s result=%s",
+                        listing_id,
+                        origin_id,
+                        image_count,
+                        result.get("upsert", {}),
+                    )
+                except Exception as exc:
+                    failed += 1
+                    logger.warning(
+                        "mobile.de image enrich failed: listing_id=%s origin_id=%s error=%s",
+                        listing_id,
+                        origin_id,
+                        exc,
+                        exc_info=True,
+                    )
+                if effective_delay:
+                    time.sleep(effective_delay)
+        return {
+            "status": "success",
+            "candidates": len(candidates),
+            "enriched": enriched,
+            "failed": failed,
+            "skipped": skipped,
+        }
+    finally:
+        _release_lock_if_owner(
+            redis_client,
+            MOBILEDE_ENRICH_IMAGES_LOCK_KEY,
+            owner_token,
+        )
 
 
 @shared_task(