start first sync
This commit is contained in:
@@ -1,9 +1,17 @@
|
|||||||
# Инициализация Celery-приложения и периодических задач для Encar.
|
# Инициализация Celery-приложения и периодических задач для Encar.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
from celery import Celery
|
from celery import Celery
|
||||||
|
from celery.signals import beat_init
|
||||||
|
from redis import Redis
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from ..core.config import settings
|
from ..core.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger("encar_scraper.worker.celery_app")
|
||||||
|
_INITIAL_SYNC_BOOTSTRAP_KEY = "encar:bootstrap:initial_sync_enqueued"
|
||||||
|
|
||||||
|
|
||||||
def _broker_url() -> str:
|
def _broker_url() -> str:
|
||||||
return settings.celery.broker_url or settings.redis.url
|
return settings.celery.broker_url or settings.redis.url
|
||||||
@@ -51,6 +59,46 @@ celery_app.conf.update(
|
|||||||
celery_app.autodiscover_tasks(["encar_scraper.worker"])
|
celery_app.autodiscover_tasks(["encar_scraper.worker"])
|
||||||
|
|
||||||
|
|
||||||
|
@beat_init.connect
|
||||||
|
def _schedule_initial_sync_on_first_start(**kwargs):
|
||||||
|
"""Ставит первый sync сразу после первого запуска проекта на пустой БД."""
|
||||||
|
from ..storage.db import PersistenceService
|
||||||
|
from ..storage.models import SyncRun
|
||||||
|
from .tasks import encar_sync_listing_task
|
||||||
|
|
||||||
|
persistence = PersistenceService(settings)
|
||||||
|
redis_client: Redis | None = None
|
||||||
|
try:
|
||||||
|
with persistence.session_scope() as session:
|
||||||
|
total_runs = session.execute(select(func.count(SyncRun.id))).scalar() or 0
|
||||||
|
|
||||||
|
if total_runs:
|
||||||
|
return
|
||||||
|
|
||||||
|
redis_client = Redis.from_url(
|
||||||
|
_broker_url(),
|
||||||
|
decode_responses=True,
|
||||||
|
socket_timeout=10,
|
||||||
|
socket_connect_timeout=5,
|
||||||
|
)
|
||||||
|
marked = redis_client.set(_INITIAL_SYNC_BOOTSTRAP_KEY, "1", nx=True, ex=600)
|
||||||
|
if not marked:
|
||||||
|
logger.info("Initial Encar sync already bootstrapped on this startup")
|
||||||
|
return
|
||||||
|
|
||||||
|
encar_sync_listing_task.apply_async(kwargs={"car_type": "all"}, queue="encar")
|
||||||
|
logger.info("Scheduled initial Encar sync because sync_runs table is empty")
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Failed to schedule initial Encar sync", exc_info=True)
|
||||||
|
finally:
|
||||||
|
persistence.engine.dispose()
|
||||||
|
if redis_client is not None:
|
||||||
|
try:
|
||||||
|
redis_client.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
from celery.signals import setup_logging as celery_setup_logging
|
from celery.signals import setup_logging as celery_setup_logging
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user