start first sync
This commit is contained in:
@@ -1,9 +1,17 @@
|
||||
# Инициализация Celery-приложения и периодических задач для Encar.
|
||||
|
||||
import logging
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import beat_init
|
||||
from redis import Redis
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..core.config import settings
|
||||
|
||||
logger = logging.getLogger("encar_scraper.worker.celery_app")
|
||||
_INITIAL_SYNC_BOOTSTRAP_KEY = "encar:bootstrap:initial_sync_enqueued"
|
||||
|
||||
|
||||
def _broker_url() -> str:
|
||||
return settings.celery.broker_url or settings.redis.url
|
||||
@@ -51,6 +59,46 @@ celery_app.conf.update(
|
||||
celery_app.autodiscover_tasks(["encar_scraper.worker"])
|
||||
|
||||
|
||||
@beat_init.connect
|
||||
def _schedule_initial_sync_on_first_start(**kwargs):
|
||||
"""Ставит первый sync сразу после первого запуска проекта на пустой БД."""
|
||||
from ..storage.db import PersistenceService
|
||||
from ..storage.models import SyncRun
|
||||
from .tasks import encar_sync_listing_task
|
||||
|
||||
persistence = PersistenceService(settings)
|
||||
redis_client: Redis | None = None
|
||||
try:
|
||||
with persistence.session_scope() as session:
|
||||
total_runs = session.execute(select(func.count(SyncRun.id))).scalar() or 0
|
||||
|
||||
if total_runs:
|
||||
return
|
||||
|
||||
redis_client = Redis.from_url(
|
||||
_broker_url(),
|
||||
decode_responses=True,
|
||||
socket_timeout=10,
|
||||
socket_connect_timeout=5,
|
||||
)
|
||||
marked = redis_client.set(_INITIAL_SYNC_BOOTSTRAP_KEY, "1", nx=True, ex=600)
|
||||
if not marked:
|
||||
logger.info("Initial Encar sync already bootstrapped on this startup")
|
||||
return
|
||||
|
||||
encar_sync_listing_task.apply_async(kwargs={"car_type": "all"}, queue="encar")
|
||||
logger.info("Scheduled initial Encar sync because sync_runs table is empty")
|
||||
except Exception:
|
||||
logger.warning("Failed to schedule initial Encar sync", exc_info=True)
|
||||
finally:
|
||||
persistence.engine.dispose()
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
from celery.signals import setup_logging as celery_setup_logging
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user