Stabilize worker sync flow
This commit is contained in:
@@ -4,11 +4,13 @@ import logging
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
|
||||
from redis import Redis
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.logs import setup_logging
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.worker.celery_app")
|
||||
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
|
||||
|
||||
|
||||
@celery_setup_logging.connect
|
||||
@@ -96,6 +98,24 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
|
||||
def _on_worker_ready(**kwargs):
|
||||
"""Сразу при старте worker отправляем первую задачу sync_listing,
|
||||
чтобы не ждать час до первого beat-цикла."""
|
||||
try:
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
|
||||
except Exception:
|
||||
logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
|
||||
return
|
||||
|
||||
if not should_dispatch:
|
||||
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
|
||||
return
|
||||
|
||||
logger.info("Worker ready — dispatching initial sync_listing task")
|
||||
celery_app.send_task(
|
||||
"iaai_scraper.worker.tasks.sync_listing_task",
|
||||
|
||||
Reference in New Issue
Block a user