Stabilize worker sync flow

This commit is contained in:
qananasikq
2026-04-17 17:30:28 +03:00
parent 3c3aea8eb3
commit 05d1ffb5ac
9 changed files with 803 additions and 80 deletions

View File

@@ -4,11 +4,13 @@ import logging
from celery import Celery
from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
from redis import Redis
from ..core.config import settings
from ..core.logs import setup_logging
logger = logging.getLogger("iaai_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
@celery_setup_logging.connect
@@ -96,6 +98,24 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
def _on_worker_ready(**kwargs):
"""Сразу при старте worker отправляем первую задачу sync_listing,
чтобы не ждать час до первого beat-цикла."""
try:
redis_client = Redis.from_url(
settings.redis.url,
decode_responses=True,
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
socket_timeout=settings.redis.socket_timeout_seconds,
health_check_interval=settings.redis.health_check_interval_seconds,
retry_on_timeout=True,
)
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
except Exception:
logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
return
if not should_dispatch:
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
return
logger.info("Worker ready — dispatching initial sync_listing task")
celery_app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",