# Инициализация Celery-приложения и периодических задач. import json import logging import os import time from celery import Celery from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging from redis import Redis from ..core.config import settings from ..core.logs import setup_logging logger = logging.getLogger("mobilede_scraper.worker.celery_app") STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched" MOBILEDE_SYNC_QUEUE = "mobilede_sync" PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:" def _env_bool(name: str, default: bool) -> bool: raw = os.getenv(name, "true" if default else "false").strip().lower() return raw in {"1", "true", "yes", "on"} def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool: now = int(time.time()) try: for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"): payload = redis_client.get(raw_key) if not payload: continue try: progress = json.loads(payload) except (TypeError, ValueError): continue ts = int(progress.get("ts") or 0) stage = str(progress.get("stage") or "") if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}: return True except Exception: logger.warning("Failed to inspect startup progress keys", exc_info=True) return False @celery_setup_logging.connect def _configure_logging(loglevel=None, **kwargs): # Перехватываем логирование Celery и пишем только в stderr (Docker logs). level = settings.log_level if settings.log_level else "INFO" setup_logging(level, None) @worker_process_init.connect def _on_worker_process_init(**kwargs): # Повторно настраиваем логирование в каждом дочернем prefork-процессе, # чтобы StreamHandler(stderr) корректно работал после fork. level = settings.log_level if settings.log_level else "INFO" setup_logging(level, None) def _broker_url() -> str: return settings.celery.broker_url or settings.redis.url def _result_backend() -> str: return settings.celery.result_backend or settings.redis.url celery_app = Celery( "mobilede_scraper", broker=_broker_url(), backend=_result_backend(), ) # Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), # ограничиваем, чтобы зависший worker не жил вечно. _soft = settings.celery.task_soft_time_limit _hard = settings.celery.task_time_limit _max_hard = _soft + 120 if _soft else _hard if _hard > _max_hard: logger.info( "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " "clamping hard limit to %d", _hard, _soft, _max_hard, ) _hard = _max_hard celery_app.conf.update( task_serializer="json", accept_content=["json"], result_serializer="json", timezone="UTC", enable_utc=True, task_soft_time_limit=_soft, task_time_limit=_hard, task_acks_late=True, task_reject_on_worker_lost=True, task_track_started=True, worker_concurrency=settings.celery.worker_concurrency, worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, worker_pool=settings.celery.worker_pool, worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, broker_transport_options={ "visibility_timeout": settings.celery.broker_visibility_timeout, }, result_expires=86400, worker_redirect_stdouts=False, worker_hijack_root_logger=False, beat_schedule={ "periodic-mobilede-sync-search": { "task": "mobilede.sync_runtime_segments", "schedule": settings.celery.beat_sync_interval_minutes * 60.0, "args": (), "kwargs": { "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), }, "options": { "queue": MOBILEDE_SYNC_QUEUE, "expires": settings.celery.beat_sync_interval_minutes * 60.0, }, } }, task_routes={ "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE}, "MOBILEDE.sync_cars_feed": {"queue": MOBILEDE_SYNC_QUEUE}, "mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE}, }, ) celery_app.autodiscover_tasks(["mobilede_scraper.worker"]) @worker_ready.connect def _on_worker_ready(**kwargs): """При старте worker отправляем первый sync_listing, если очередь пуста.""" if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True): logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED") return redis_client = None try: redis_client = Redis.from_url( settings.redis.url, decode_responses=True, socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, socket_timeout=settings.redis.socket_timeout_seconds, health_check_interval=settings.redis.health_check_interval_seconds, retry_on_timeout=True, ) has_fresh_progress = _has_fresh_active_progress(redis_client) for stale_key in ("mobilede:locks:sync_listing",): try: ttl = redis_client.ttl(stale_key) if ttl is not None and ttl != -2 and not has_fresh_progress: redis_client.delete(stale_key) logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) elif ttl is not None and ttl != -2: logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl) except Exception: logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True) try: queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0) except Exception: queue_len = 0 if queue_len > 0: logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len) return should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) if not should_dispatch and not has_fresh_progress: redis_client.delete(STARTUP_SYNC_DISPATCH_KEY) should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) if should_dispatch: logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists") except Exception: logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) return finally: if redis_client is not None: try: redis_client.close() except Exception: pass if not should_dispatch: logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") return logger.info("Worker ready — dispatching initial mobile.de sync_search task") celery_app.send_task( "mobilede.sync_runtime_segments", kwargs={ "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), }, queue=MOBILEDE_SYNC_QUEUE, expires=settings.celery.beat_sync_interval_minutes * 60.0, )