IAAI scraper: Playwright + SQLAlchemy, парсинг авто с аукциона, Docker-ready
This commit is contained in:
31
iaai_scraper/core/retry.py
Normal file
31
iaai_scraper/core/retry.py
Normal file
@@ -0,0 +1,31 @@
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.retry")
|
||||
|
||||
|
||||
def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
||||
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
||||
@wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc:
|
||||
last_error = exc
|
||||
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
|
||||
if attempt < max_attempts:
|
||||
time.sleep(delay_seconds * (2 ** (attempt - 1)))
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError("Retry wrapper failed without a captured exception")
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
Reference in New Issue
Block a user