54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
import logging
|
|
import random
|
|
import time
|
|
from collections.abc import Callable
|
|
from functools import wraps
|
|
from typing import Any
|
|
|
|
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
|
|
|
|
from .exceptions import AntiBotDetectedError
|
|
|
|
logger = logging.getLogger("iaai_scraper.retry")
|
|
|
|
# Типы исключений, при которых retry имеет смысл.
|
|
RETRYABLE_EXCEPTIONS = (
|
|
PlaywrightTimeoutError,
|
|
Error,
|
|
ConnectionError,
|
|
OSError,
|
|
TimeoutError,
|
|
AntiBotDetectedError,
|
|
)
|
|
|
|
|
|
def retryable(
|
|
max_attempts: int,
|
|
delay_seconds: float = 2.5,
|
|
backoff_multiplier: float = 2.0,
|
|
jitter_seconds: float = 0.0,
|
|
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
|
@wraps(func)
|
|
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
last_error: Exception | None = None
|
|
for attempt in range(1, max_attempts + 1):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except RETRYABLE_EXCEPTIONS as exc:
|
|
last_error = exc
|
|
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
|
|
if attempt < max_attempts:
|
|
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
|
|
if jitter_seconds > 0:
|
|
sleep_for += random.uniform(0, jitter_seconds)
|
|
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
|
|
time.sleep(sleep_for)
|
|
if last_error is not None:
|
|
raise last_error
|
|
raise RuntimeError("Retry wrapper failed without a captured exception")
|
|
|
|
return wrapper
|
|
|
|
return decorator
|