fix worker recovery and docker logs

iaai_scraper/worker/tasks.py

@@ -18,6 +18,28 @@ logger = logging.getLogger("iaai_scraper.worker.tasks")
 SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
 
 
+def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
+    last_exc: Exception | None = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return func()
+        except Exception as exc:
+            last_exc = exc
+            if attempt >= attempts:
+                break
+            delay = base_delay_s * (2 ** (attempt - 1))
+            logger.warning(
+                "Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
+                attempt,
+                attempts,
+                exc,
+                delay,
+            )
+            time.sleep(delay)
+    if last_exc is not None:
+        raise last_exc
+
+
 def _run_browser_job(func, *args, **kwargs):
     # Браузерный код запускаем в отдельном потоке без активного loop.
     with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor:
@@ -32,17 +54,42 @@ def _sync_listing_lock_ttl_seconds() -> int:
 
 
 def _get_persistence() -> PersistenceService:
-    return PersistenceService(Settings())
+    settings = Settings()
+    persistence = PersistenceService(settings)
+
+    def _ping_db() -> None:
+        with persistence.engine.connect() as conn:
+            conn.exec_driver_sql("SELECT 1")
+
+    _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
+    return persistence
 
 
 def _get_redis() -> Redis:
     settings = Settings()
-    return Redis.from_url(settings.redis.url, decode_responses=True)
+    redis_client = Redis.from_url(settings.redis.url, decode_responses=True)
+
+    def _ping_redis() -> None:
+        redis_client.ping()
+
+    _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
+    return redis_client
 
 
 def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
     try:
-        return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+        acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+        if acquired:
+            return True
+
+        # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём.
+        ttl = redis_client.ttl(key)
+        if ttl is not None and ttl < 0:
+            logger.warning("Detected stale lock without TTL, removing: %s", key)
+            redis_client.delete(key)
+            return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+
+        return False
     except Exception as exc:
         logger.warning("Failed to acquire lock %s", key, exc_info=True)
         return False