fast parser

iaai_scraper/worker/celery_app.py

@@ -1,7 +1,9 @@
 # Инициализация Celery-приложения и периодических задач.
 
+import json
 import logging
 import os
+import time
 
 from celery import Celery
 from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
@@ -13,6 +15,7 @@ from ..core.logs import setup_logging
 logger = logging.getLogger("iaai_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 IAAI_SYNC_QUEUE = "iaai_sync"
+PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
 
 
 def _env_bool(name: str, default: bool) -> bool:
@@ -20,6 +23,26 @@ def _env_bool(name: str, default: bool) -> bool:
     return raw in {"1", "true", "yes", "on"}
 
 
+def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
+    now = int(time.time())
+    try:
+        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
+            payload = redis_client.get(raw_key)
+            if not payload:
+                continue
+            try:
+                progress = json.loads(payload)
+            except (TypeError, ValueError):
+                continue
+            ts = int(progress.get("ts") or 0)
+            stage = str(progress.get("stage") or "")
+            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
+                return True
+    except Exception:
+        logger.warning("Failed to inspect startup progress keys", exc_info=True)
+    return False
+
+
 @celery_setup_logging.connect
 def _configure_logging(loglevel=None, **kwargs):
     # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
@@ -55,7 +78,7 @@ _soft = settings.celery.task_soft_time_limit
 _hard = settings.celery.task_time_limit
 _max_hard = _soft + 120 if _soft else _hard
 if _hard > _max_hard:
-    logger.warning(
+    logger.info(
         "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
         "clamping hard limit to %d",
         _hard, _soft, _max_hard,
@@ -75,7 +98,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="solo",
+    worker_pool=settings.celery.worker_pool,
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -123,14 +146,17 @@ def _on_worker_ready(**kwargs):
             retry_on_timeout=True,
         )
 
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
         for stale_key in ("iaai:locks:sync_listing",):
             try:
                 ttl = redis_client.ttl(stale_key)
-                if ttl is not None and ttl != -2:
+                if ttl is not None and ttl != -2 and not has_fresh_progress:
                     redis_client.delete(stale_key)
-                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                    logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                elif ttl is not None and ttl != -2:
+                    logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
             except Exception:
-                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+                logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
 
         try:
             queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
@@ -141,6 +167,11 @@ def _on_worker_ready(**kwargs):
             return
 
         should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        if not should_dispatch and not has_fresh_progress:
+            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+            if should_dispatch:
+                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return