Initial project version

iaai_scraper/worker/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import celery_app
+
+__all__ = ["celery_app"]

iaai_scraper/worker/celery_app.py

@@ -0,0 +1,51 @@
+from celery import Celery
+
+from ..core.config import settings
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "iaai_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+
+    task_soft_time_limit=settings.celery.task_soft_time_limit,
+    task_time_limit=settings.celery.task_time_limit,
+    worker_concurrency=settings.celery.worker_concurrency,
+
+    # prefork + 1 процесс, тк playwright sync api
+    worker_pool="prefork",
+    worker_prefetch_multiplier=1,
+
+    broker_connection_retry_on_startup=True,
+
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "kwargs": {"limit": settings.celery.beat_sync_limit},
+            "options": {"queue": "scraping"},
+        },
+    },
+
+    task_routes={
+        "iaai_scraper.worker.tasks.*": {"queue": "scraping"},
+    },
+)
+
+celery_app.autodiscover_tasks(["iaai_scraper.worker"])

iaai_scraper/worker/tasks.py

@@ -0,0 +1,133 @@
+import json
+import logging
+from datetime import datetime, timezone
+
+from celery import shared_task
+
+from ..core.config import Settings
+from ..scraper import IAAIScraper
+from ..storage.db import PersistenceService
+
+logger = logging.getLogger("iaai_scraper.worker.tasks")
+
+
+def _get_persistence() -> PersistenceService:
+    return PersistenceService(Settings())
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    bind=True,
+    max_retries=2,
+    default_retry_delay=30,
+    acks_late=True,
+)
+def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
+    task_id = self.request.id
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    # регистрируем запуск
+    persistence.update_scrape_task(
+        task_id,
+        status="running",
+        started_at=datetime.now(timezone.utc),
+    )
+
+    try:
+        with IAAIScraper() as scraper:
+            result = scraper.sync_vehicle(vehicle_url, lane=lane)
+
+        persistence.update_scrape_task(
+            task_id,
+            status="success",
+            finished_at=datetime.now(timezone.utc),
+            result_summary=json.dumps({
+                "car_upserted": True,
+                "db_action": result.get("db_action"),
+                "images_upserted": result.get("images_upserted", 0),
+                "elapsed_seconds": result.get("elapsed_seconds"),
+            }, default=str),
+        )
+        logger.info("sync_vehicle_task completed: %s", vehicle_url)
+        return {
+            "status": "success",
+            "vehicle_url": vehicle_url,
+            "db_action": result.get("db_action"),
+            "images_upserted": result.get("images_upserted", 0),
+        }
+
+    except Exception as exc:
+        persistence.update_scrape_task(
+            task_id,
+            status="failed",
+            finished_at=datetime.now(timezone.utc),
+            error_message=str(exc)[:2000],
+        )
+        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
+        raise self.retry(exc=exc)
+
+
+@shared_task(
+    name="iaai_scraper.worker.tasks.sync_listing_task",
+    bind=True,
+    max_retries=1,
+    default_retry_delay=120,
+    acks_late=True,
+)
+def sync_listing_task(
+    self,
+    make: str | None = None,
+    model: str | None = None,
+    lane: str = "iaai_cars",
+    limit: int | None = None,
+    only_new: bool | None = None,
+):
+    task_id = self.request.id
+    persistence = _get_persistence()
+    persistence.create_tables()
+
+    persistence.update_scrape_task(
+        task_id,
+        status="running",
+        started_at=datetime.now(timezone.utc),
+    )
+
+    try:
+        with IAAIScraper() as scraper:
+            result = scraper.sync_listing(
+                make=make,
+                model=model,
+                lane=lane,
+                limit=limit,
+                only_new=only_new,
+            )
+
+        summary = {
+            "cars_upserted": result.get("cars_upserted", 0),
+            "cars_failed": result.get("cars_failed", 0),
+            "images_upserted": result.get("images_upserted", 0),
+            "skipped_existing": result.get("skipped_existing", 0),
+            "elapsed_seconds": result.get("elapsed_seconds"),
+        }
+        persistence.update_scrape_task(
+            task_id,
+            status="success",
+            finished_at=datetime.now(timezone.utc),
+            result_summary=json.dumps(summary, default=str),
+        )
+        logger.info(
+            "sync_listing_task completed: %d upserted, %d failed",
+            summary["cars_upserted"], summary["cars_failed"],
+        )
+        return {"status": "success", **summary}
+
+    except Exception as exc:
+        persistence.update_scrape_task(
+            task_id,
+            status="failed",
+            finished_at=datetime.now(timezone.utc),
+            error_message=str(exc)[:2000],
+        )
+        logger.error("sync_listing_task failed: %s", exc)
+        raise self.retry(exc=exc)