add segmented listing sync

iaai_scraper/worker/tasks.py

@@ -11,7 +11,7 @@ from billiard.exceptions import SoftTimeLimitExceeded
 from celery import shared_task
 from redis import Redis
 
-from ..core.config import Settings
+from ..core.config import Settings, parse_listing_segments
 from ..scraper import IAAIScraper
 from ..storage.db import PersistenceService
 
@@ -166,7 +166,7 @@ def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> N
         logger.warning("Failed to release lock %s", key, exc_info=True)
 
 
-def _has_running_sync_listing_tasks(celery_app) -> bool:
+def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
     try:
         inspector = celery_app.control.inspect(timeout=1.0)
         snapshots = [
@@ -182,12 +182,17 @@ def _has_running_sync_listing_tasks(celery_app) -> bool:
         for entries in snapshot.values():
             for entry in entries or []:
                 task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
-                if task_name == SYNC_LISTING_TASK_NAME:
-                    return True
+                if task_name != SYNC_LISTING_TASK_NAME:
+                    continue
+                # Исключаем текущую задачу — она не считается "другой запущенной"
+                entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
+                if exclude_task_id and entry_id == exclude_task_id:
+                    continue
+                return True
     return False
 
 
-def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app) -> bool:
+def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
     try:
         owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
         if not owner_token:
@@ -196,7 +201,7 @@ def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app) -> bool:
         logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
         return False
 
-    if _has_running_sync_listing_tasks(celery_app):
+    if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
         logger.info("sync_listing lock preserved: active task still detected")
         return False
 
@@ -256,6 +261,7 @@ def _save_sync_checkpoint(
     make: str | None,
     model: str | None,
     lane: str,
+    segment_index: int | None = None,
 ) -> None:
     payload = {
         "status": "in_progress",
@@ -264,6 +270,7 @@ def _save_sync_checkpoint(
         "make": make,
         "model": model,
         "lane": lane,
+        "segment_index": segment_index,
         "updated_at": int(time.time()),
     }
     try:
@@ -384,7 +391,7 @@ def sync_listing_task(
     lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
 
     if not lock_acquired:
-        orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app)
+        orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
         if orphan_cleared:
             lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
 
@@ -409,16 +416,20 @@ def sync_listing_task(
             checkpoint_make = checkpoint.get("make")
             checkpoint_model = checkpoint.get("model")
             checkpoint_lane = checkpoint.get("lane")
+            checkpoint_segment = checkpoint.get("segment_index")
             same_scope = (
                 checkpoint_make == make
                 and checkpoint_model == model
                 and checkpoint_lane == lane
             )
-            if checkpoint_page > 0 and same_scope:
+            # Для сегментированного режима: совпадение по lane + наличие segment_index
+            is_segmented_checkpoint = checkpoint_segment is not None and checkpoint_lane == lane
+            if checkpoint_page > 0 and (same_scope or is_segmented_checkpoint):
                 resume_from_page = checkpoint_page + 1
                 logger.warning(
-                    "Resuming sync_listing from page %d using checkpoint",
+                    "Resuming sync_listing from page %d (segment=%s) using checkpoint",
                     resume_from_page,
+                    checkpoint_segment,
                 )
             elif checkpoint_page > 0:
                 logger.info("Ignoring stale checkpoint due to different sync parameters")
@@ -442,8 +453,38 @@ def sync_listing_task(
             lock_ttl,
         )
         self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
+
+        # Определяем сегменты из конфига.
+        settings = Settings()
+        segments = parse_listing_segments(settings.listing.listing_segments_json)
+        use_segmented = bool(segments) and make is None and model is None
+
+        resume_from_segment = 0
+        if use_segmented and checkpoint and str(checkpoint.get("status") or "") == "in_progress":
+            cp_segment = checkpoint.get("segment_index")
+            if cp_segment is not None and int(cp_segment) >= 0:
+                resume_from_segment = int(cp_segment)
+                # resume_from_page уже вычислен выше
+
         def _job():
             with IAAIScraper() as scraper:
+                if use_segmented:
+                    return scraper.sync_listing_segmented(
+                        segments=segments,
+                        lane=lane,
+                        only_new=effective_only_new,
+                        start_segment=resume_from_segment,
+                        start_page=resume_from_page,
+                        progress_callback=lambda seg_idx, page_number: _save_sync_checkpoint(
+                            redis_client,
+                            task_id=task_id,
+                            page_number=page_number,
+                            make=None,
+                            model=None,
+                            lane=lane,
+                            segment_index=seg_idx,
+                        ),
+                    )
                 return scraper.sync_listing(
                     make=make,
                     model=model,