Prepare mobile de parser release
This commit is contained in:
qananasikq
207
iaai_scraper/worker/celery_app.py
Normal file
207
iaai_scraper/worker/celery_app.py
Normal file
@@ -0,0 +1,207 @@
|
||||
# Инициализация Celery-приложения и периодических задач.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
|
||||
from redis import Redis
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.logs import setup_logging
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.worker.celery_app")
|
||||
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
|
||||
IAAI_SYNC_QUEUE = "iaai_sync"
|
||||
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
|
||||
PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.getenv(name, "true" if default else "false").strip().lower()
|
||||
return raw in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
|
||||
now = int(time.time())
|
||||
try:
|
||||
for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
|
||||
payload = redis_client.get(raw_key)
|
||||
if not payload:
|
||||
continue
|
||||
try:
|
||||
progress = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
ts = int(progress.get("ts") or 0)
|
||||
stage = str(progress.get("stage") or "")
|
||||
if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("Failed to inspect startup progress keys", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
@celery_setup_logging.connect
|
||||
def _configure_logging(loglevel=None, **kwargs):
|
||||
# Перехватываем логирование Celery и пишем только в stderr (Docker logs).
|
||||
level = settings.log_level if settings.log_level else "INFO"
|
||||
setup_logging(level, None)
|
||||
|
||||
|
||||
@worker_process_init.connect
|
||||
def _on_worker_process_init(**kwargs):
|
||||
# Повторно настраиваем логирование в каждом дочернем prefork-процессе,
|
||||
# чтобы StreamHandler(stderr) корректно работал после fork.
|
||||
level = settings.log_level if settings.log_level else "INFO"
|
||||
setup_logging(level, None)
|
||||
|
||||
|
||||
def _broker_url() -> str:
|
||||
return settings.celery.broker_url or settings.redis.url
|
||||
|
||||
|
||||
def _result_backend() -> str:
|
||||
return settings.celery.result_backend or settings.redis.url
|
||||
|
||||
|
||||
celery_app = Celery(
|
||||
"iaai_scraper",
|
||||
broker=_broker_url(),
|
||||
backend=_result_backend(),
|
||||
)
|
||||
|
||||
# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
|
||||
# ограничиваем, чтобы зависший worker не жил вечно.
|
||||
_soft = settings.celery.task_soft_time_limit
|
||||
_hard = settings.celery.task_time_limit
|
||||
_max_hard = _soft + 120 if _soft else _hard
|
||||
if _hard > _max_hard:
|
||||
logger.info(
|
||||
"CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
|
||||
"clamping hard limit to %d",
|
||||
_hard, _soft, _max_hard,
|
||||
)
|
||||
_hard = _max_hard
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_serializer="json",
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_soft_time_limit=_soft,
|
||||
task_time_limit=_hard,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
task_track_started=True,
|
||||
worker_concurrency=settings.celery.worker_concurrency,
|
||||
worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
|
||||
worker_pool=settings.celery.worker_pool,
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
broker_transport_options={
|
||||
"visibility_timeout": settings.celery.broker_visibility_timeout,
|
||||
},
|
||||
result_expires=86400,
|
||||
worker_redirect_stdouts=False,
|
||||
worker_hijack_root_logger=False,
|
||||
beat_schedule={
|
||||
"periodic-mobilede-sync-search": {
|
||||
"task": "mobilede.sync_runtime_segments",
|
||||
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
"args": (),
|
||||
"kwargs": {
|
||||
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
|
||||
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
|
||||
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
|
||||
},
|
||||
"options": {
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
},
|
||||
}
|
||||
},
|
||||
task_routes={
|
||||
"mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
|
||||
"iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
|
||||
},
|
||||
)
|
||||
|
||||
celery_app.autodiscover_tasks(["iaai_scraper.worker"])
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _on_worker_ready(**kwargs):
|
||||
"""При старте worker отправляем первый sync_listing, если очередь пуста."""
|
||||
if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
|
||||
logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
|
||||
return
|
||||
|
||||
redis_client = None
|
||||
try:
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
|
||||
has_fresh_progress = _has_fresh_active_progress(redis_client)
|
||||
for stale_key in ("iaai:locks:sync_listing",):
|
||||
try:
|
||||
ttl = redis_client.ttl(stale_key)
|
||||
if ttl is not None and ttl != -2 and not has_fresh_progress:
|
||||
redis_client.delete(stale_key)
|
||||
logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
|
||||
elif ttl is not None and ttl != -2:
|
||||
logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
|
||||
except Exception:
|
||||
logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
|
||||
|
||||
try:
|
||||
queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
|
||||
except Exception:
|
||||
queue_len = 0
|
||||
if queue_len > 0:
|
||||
logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
|
||||
return
|
||||
|
||||
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
|
||||
if not should_dispatch and not has_fresh_progress:
|
||||
redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
|
||||
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
|
||||
if should_dispatch:
|
||||
logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
|
||||
except Exception:
|
||||
logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
|
||||
return
|
||||
finally:
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not should_dispatch:
|
||||
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
|
||||
return
|
||||
|
||||
logger.info("Worker ready — dispatching initial mobile.de sync_search task")
|
||||
celery_app.send_task(
|
||||
"mobilede.sync_runtime_segments",
|
||||
kwargs={
|
||||
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
|
||||
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
|
||||
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
|
||||
},
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
expires=settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
)
|
||||
Reference in New Issue
Block a user