Prepared for production

iaai_scraper/worker/tasks.py

@@ -19,6 +19,8 @@ from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitema
 
 logger = logging.getLogger("iaai_scraper.worker.tasks")
 
+IAAI_SYNC_QUEUE = "iaai_sync"
+
 # Минимальная пауза между батчами (секунды) — не давит IAAI.
 _INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0)
 # Если доля failed в батче превышает порог — прерываем (IAAI блокирует).
@@ -44,7 +46,7 @@ HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak"
 HOURLY_FAILURE_STREAK_LIMIT = 3
 HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60  # сброс через 6 часов
 SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
-SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
+SYNC_LISTING_TASK_NAME = "iaai.sync_cars_feed"
 SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
 SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
 SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
@@ -482,7 +484,7 @@ def _start_stall_watchdog(
                     celery_app.send_task(
                         SYNC_LISTING_TASK_NAME,
                         kwargs={},
-                        queue="scraping",
+                        queue=IAAI_SYNC_QUEUE,
                         countdown=15,
                         expires=followup_ttl,
                     )
@@ -903,6 +905,7 @@ def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[in
 
 @shared_task(
     name="iaai_scraper.worker.tasks.sync_segment_task",
+    queue=IAAI_SYNC_QUEUE,
     bind=True,
     max_retries=2,
     default_retry_delay=60,
@@ -1076,6 +1079,7 @@ def sync_segment_task(
 
 @shared_task(
     name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    queue=IAAI_SYNC_QUEUE,
     bind=True,
     max_retries=2,
     default_retry_delay=30,
@@ -1106,7 +1110,8 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
 
 
 @shared_task(
-    name="iaai_scraper.worker.tasks.sync_listing_task",
+    name=SYNC_LISTING_TASK_NAME,
+    queue=IAAI_SYNC_QUEUE,
     bind=True,
     max_retries=3,
     default_retry_delay=120,
@@ -1178,7 +1183,7 @@ def sync_listing_task(
         try:
             followup_expires = max(int(lock_ttl), int(delay_seconds) + 300)
             self.app.send_task(
-                "iaai_scraper.worker.tasks.sync_listing_task",
+                SYNC_LISTING_TASK_NAME,
                 kwargs={
                     "make": make,
                     "model": model,
@@ -1186,7 +1191,7 @@ def sync_listing_task(
                     "limit": limit,
                     "only_new": only_new,
                 },
-                queue="scraping",
+                queue=IAAI_SYNC_QUEUE,
                 countdown=max(0, int(delay_seconds)),
                 expires=followup_expires,
             )
@@ -1480,7 +1485,7 @@ def sync_listing_task(
                             "only_new": effective_only_new,
                             "is_bootstrap": force_bootstrap_full_scan,
                         },
-                        queue="scraping",
+                        queue=IAAI_SYNC_QUEUE,
                     )
                     dispatched += 1
                 except Exception:
@@ -1654,7 +1659,7 @@ def sync_listing_task(
                 if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl):
                     try:
                         self.app.send_task(
-                            "iaai_scraper.worker.tasks.sync_listing_task",
+                            SYNC_LISTING_TASK_NAME,
                             kwargs={
                                 "make": make,
                                 "model": model,
@@ -1662,7 +1667,7 @@ def sync_listing_task(
                                 "limit": limit,
                                 "only_new": only_new,
                             },
-                            queue="scraping",
+                            queue=IAAI_SYNC_QUEUE,
                             countdown=10,
                             expires=followup_ttl,
                         )