Track sold cars

mobilede_scraper/worker/celery_app.py

@@ -11,6 +11,7 @@ from redis import Redis
 
 from ..core.config import settings
 from ..core.logs import setup_logging
+from .constants import GLOBAL_DB_PROGRESS_TS_KEY, GLOBAL_PROGRESS_TS_KEY
 
 logger = logging.getLogger("mobilede_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
@@ -23,6 +24,9 @@ def _env_bool(name: str, default: bool) -> bool:
     return raw in {"1", "true", "yes", "on"}
 
 
+MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
+
+
 def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
     now = int(time.time())
     try:
@@ -43,6 +47,18 @@ def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 18
     return False
 
 
+def _has_recent_global_progress(redis_client: Redis, *, max_age_seconds: int = 300) -> bool:
+    now = int(time.time())
+    try:
+        progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
+        db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
+    except Exception:
+        logger.warning("Failed to inspect global startup progress keys", exc_info=True)
+        return False
+    freshest_ts = max(progress_ts, db_progress_ts)
+    return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
+
+
 @celery_setup_logging.connect
 def _configure_logging(loglevel=None, **kwargs):
     # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
@@ -85,6 +101,25 @@ if _hard > _max_hard:
     )
     _hard = _max_hard
 
+beat_schedule = {}
+if MOBILEDE_BEAT_SYNC_ENABLED:
+    beat_schedule = {
+        "periodic-mobilede-sync-search": {
+            "task": "mobilede.sync_runtime_segments",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {
+                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+            },
+            "options": {
+                "queue": MOBILEDE_SYNC_QUEUE,
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
+        }
+    }
+
 celery_app.conf.update(
     task_serializer="json",
     accept_content=["json"],
@@ -107,22 +142,7 @@ celery_app.conf.update(
     result_expires=86400,
     worker_redirect_stdouts=False,
     worker_hijack_root_logger=False,
-    beat_schedule={
-        "periodic-mobilede-sync-search": {
-            "task": "mobilede.sync_runtime_segments",
-            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
-            "args": (),
-            "kwargs": {
-                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
-                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
-                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
-            },
-            "options": {
-                "queue": MOBILEDE_SYNC_QUEUE,
-                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
-            },
-        }
-    },
+    beat_schedule=beat_schedule,
     task_routes={
         "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
         "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
@@ -153,17 +173,24 @@ def _on_worker_ready(**kwargs):
         )
 
         has_fresh_progress = _has_fresh_active_progress(redis_client)
+        has_recent_global_progress = _has_recent_global_progress(redis_client)
+        has_live_progress = bool(has_fresh_progress or has_recent_global_progress)
 
         try:
             queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
         except Exception:
             queue_len = 0
-        if queue_len > 0:
+        if queue_len > 0 and has_live_progress:
             logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
             return
+        if queue_len > 0 and not has_live_progress:
+            logger.warning(
+                "Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
+                queue_len,
+            )
 
         should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
-        if not should_dispatch and not has_fresh_progress:
+        if not should_dispatch and not has_live_progress:
             redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
             should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
             if should_dispatch:
@@ -182,7 +209,7 @@ def _on_worker_ready(**kwargs):
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
         return
 
-    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
+    logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
     celery_app.send_task(
         "mobilede.sync_runtime_segments",
         kwargs={