cleanup and fix runtime bugs
This commit is contained in:
@@ -1,8 +1,7 @@
|
||||
"""Celery tasks для IAAI."""
|
||||
# Celery-задачи для синхронизации автомобилей и листинга IAAI.
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from celery import shared_task
|
||||
|
||||
@@ -25,33 +24,13 @@ def _get_persistence() -> PersistenceService:
|
||||
acks_late=True,
|
||||
)
|
||||
def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
|
||||
"""Скрапинг и upsert одного автомобиля."""
|
||||
task_id = self.request.id
|
||||
# Скрапинг и upsert одного автомобиля.
|
||||
persistence = _get_persistence()
|
||||
persistence.create_tables()
|
||||
|
||||
# Регистрируем запуск.
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="running",
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
try:
|
||||
with IAAIScraper() as scraper:
|
||||
result = scraper.sync_vehicle(vehicle_url, lane=lane)
|
||||
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="success",
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
result_summary=json.dumps({
|
||||
"car_upserted": True,
|
||||
"db_action": result.get("db_action"),
|
||||
"images_upserted": result.get("images_upserted", 0),
|
||||
"elapsed_seconds": result.get("elapsed_seconds"),
|
||||
}, default=str),
|
||||
)
|
||||
logger.info("sync_vehicle_task completed: %s", vehicle_url)
|
||||
return {
|
||||
"status": "success",
|
||||
@@ -61,12 +40,6 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="failed",
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
|
||||
raise self.retry(exc=exc)
|
||||
|
||||
@@ -86,17 +59,10 @@ def sync_listing_task(
|
||||
limit: int | None = None,
|
||||
only_new: bool | None = None,
|
||||
):
|
||||
"""Полный цикл: листинг + sync всех найденных машин."""
|
||||
task_id = self.request.id
|
||||
# Полный цикл: листинг + sync всех найденных машин.
|
||||
persistence = _get_persistence()
|
||||
persistence.create_tables()
|
||||
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="running",
|
||||
started_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
try:
|
||||
with IAAIScraper() as scraper:
|
||||
result = scraper.sync_listing(
|
||||
@@ -114,12 +80,6 @@ def sync_listing_task(
|
||||
"skipped_existing": result.get("skipped_existing", 0),
|
||||
"elapsed_seconds": result.get("elapsed_seconds"),
|
||||
}
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="success",
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
result_summary=json.dumps(summary, default=str),
|
||||
)
|
||||
logger.info(
|
||||
"sync_listing_task completed: %d upserted, %d failed",
|
||||
summary["cars_upserted"], summary["cars_failed"],
|
||||
@@ -127,11 +87,5 @@ def sync_listing_task(
|
||||
return {"status": "success", **summary}
|
||||
|
||||
except Exception as exc:
|
||||
persistence.update_scrape_task(
|
||||
task_id,
|
||||
status="failed",
|
||||
finished_at=datetime.now(timezone.utc),
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
logger.error("sync_listing_task failed: %s", exc)
|
||||
raise self.retry(exc=exc)
|
||||
|
||||
Reference in New Issue
Block a user