cleanup and fix runtime bugs

iaai_scraper/worker/celery_app.py

@@ -1,4 +1,4 @@
-"""Celery app factory."""
+# Инициализация Celery-приложения и периодических задач.
 
 from celery import Celery
 
@@ -20,26 +20,25 @@ celery_app = Celery(
 )
 
 celery_app.conf.update(
-    # Сериализация.
     task_serializer="json",
     accept_content=["json"],
     result_serializer="json",
     timezone="UTC",
     enable_utc=True,
-
-    # Лимиты.
     task_soft_time_limit=settings.celery.task_soft_time_limit,
     task_time_limit=settings.celery.task_time_limit,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
-
-    # Playwright sync API: prefork + 1 процесс.
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
     worker_pool="prefork",
     worker_prefetch_multiplier=1,
-
-    # Retry policy брокера.
     broker_connection_retry_on_startup=True,
-
-    # Beat schedule.
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
     beat_schedule={
         "periodic-sync-listing": {
             "task": "iaai_scraper.worker.tasks.sync_listing_task",
@@ -47,14 +46,11 @@ celery_app.conf.update(
             "args": (),
             "kwargs": {"limit": settings.celery.beat_sync_limit},
             "options": {"queue": "scraping"},
-        },
+        }
     },
-
-    # Роутинг задач.
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"},
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
     },
 )
 
-# Автообнаружение задач.
-celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])

iaai_scraper/worker/tasks.py

@@ -1,8 +1,7 @@
-"""Celery tasks для IAAI."""
+# Celery-задачи для синхронизации автомобилей и листинга IAAI.
 
 import json
 import logging
-from datetime import datetime, timezone
 
 from celery import shared_task
 
@@ -25,33 +24,13 @@ def _get_persistence() -> PersistenceService:
     acks_late=True,
 )
 def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
-    """Скрапинг и upsert одного автомобиля."""
-    task_id = self.request.id
+    # Скрапинг и upsert одного автомобиля.
     persistence = _get_persistence()
     persistence.create_tables()
 
-    # Регистрируем запуск.
-    persistence.update_scrape_task(
-        task_id,
-        status="running",
-        started_at=datetime.now(timezone.utc),
-    )
-
     try:
         with IAAIScraper() as scraper:
             result = scraper.sync_vehicle(vehicle_url, lane=lane)
-
-        persistence.update_scrape_task(
-            task_id,
-            status="success",
-            finished_at=datetime.now(timezone.utc),
-            result_summary=json.dumps({
-                "car_upserted": True,
-                "db_action": result.get("db_action"),
-                "images_upserted": result.get("images_upserted", 0),
-                "elapsed_seconds": result.get("elapsed_seconds"),
-            }, default=str),
-        )
         logger.info("sync_vehicle_task completed: %s", vehicle_url)
         return {
             "status": "success",
@@ -61,12 +40,6 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
         }
 
     except Exception as exc:
-        persistence.update_scrape_task(
-            task_id,
-            status="failed",
-            finished_at=datetime.now(timezone.utc),
-            error_message=str(exc)[:2000],
-        )
         logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
         raise self.retry(exc=exc)
 
@@ -86,17 +59,10 @@ def sync_listing_task(
     limit: int | None = None,
     only_new: bool | None = None,
 ):
-    """Полный цикл: листинг + sync всех найденных машин."""
-    task_id = self.request.id
+    # Полный цикл: листинг + sync всех найденных машин.
     persistence = _get_persistence()
     persistence.create_tables()
 
-    persistence.update_scrape_task(
-        task_id,
-        status="running",
-        started_at=datetime.now(timezone.utc),
-    )
-
     try:
         with IAAIScraper() as scraper:
             result = scraper.sync_listing(
@@ -114,12 +80,6 @@ def sync_listing_task(
             "skipped_existing": result.get("skipped_existing", 0),
             "elapsed_seconds": result.get("elapsed_seconds"),
         }
-        persistence.update_scrape_task(
-            task_id,
-            status="success",
-            finished_at=datetime.now(timezone.utc),
-            result_summary=json.dumps(summary, default=str),
-        )
         logger.info(
             "sync_listing_task completed: %d upserted, %d failed",
             summary["cars_upserted"], summary["cars_failed"],
@@ -127,11 +87,5 @@ def sync_listing_task(
         return {"status": "success", **summary}
 
     except Exception as exc:
-        persistence.update_scrape_task(
-            task_id,
-            status="failed",
-            finished_at=datetime.now(timezone.utc),
-            error_message=str(exc)[:2000],
-        )
         logger.error("sync_listing_task failed: %s", exc)
         raise self.retry(exc=exc)