improve scraper runtime

iaai_scraper/core/retry.py

@@ -1,4 +1,5 @@
 import logging
+import random
 import time
 from collections.abc import Callable
 from functools import wraps
@@ -8,8 +9,23 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
 logger = logging.getLogger("iaai_scraper.retry")
 
+# Ошибки, которые считаем временными и пригодными для повтора.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+)
 
-def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    # Универсальный retry с backoff и небольшим случайным jitter.
     def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
         @wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -17,11 +33,15 @@ def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callab
             for attempt in range(1, max_attempts + 1):
                 try:
                     return func(*args, **kwargs)
-                except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc:
+                except RETRYABLE_EXCEPTIONS as exc:
                     last_error = exc
                     logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
                     if attempt < max_attempts:
-                        time.sleep(delay_seconds * (2 ** (attempt - 1)))
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
             if last_error is not None:
                 raise last_error
             raise RuntimeError("Retry wrapper failed without a captured exception")