fix scraper flow

iaai_scraper/worker/celery_app.py

@@ -68,7 +68,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="prefork",
+    worker_pool="solo",
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -82,8 +82,11 @@ celery_app.conf.update(
             "task": "iaai_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
-            "options": {"queue": "scraping"},
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "options": {
+                "queue": "scraping",
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
         }
     },
     task_routes={
@@ -96,8 +99,8 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
-    """Сразу при старте worker отправляем первую задачу sync_listing,
-    чтобы не ждать час до первого beat-цикла."""
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    redis_client = None
     try:
         redis_client = Redis.from_url(
             settings.redis.url,
@@ -107,10 +110,34 @@ def _on_worker_ready(**kwargs):
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
+
+        for stale_key in ("iaai:locks:sync_listing",):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2:
+                    redis_client.delete(stale_key)
+                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen("scraping") or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
         should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
 
     if not should_dispatch:
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
@@ -121,4 +148,5 @@ def _on_worker_ready(**kwargs):
         "iaai_scraper.worker.tasks.sync_listing_task",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
         queue="scraping",
-    )
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+    )