improve scraper runtime

2026-04-08 13:30:45 +03:00
parent 21bdf17f35
commit f96e5ca5f8
9 changed files with 207 additions and 21 deletions
--- a/iaai_scraper/core/config.py
+++ b/iaai_scraper/core/config.py
@@ -9,6 +9,7 @@ load_dotenv()

@dataclass(slots=True)
 class FingerprintConfig:
+    # Настройки браузерного отпечатка.
    user_agent: str = (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,6 +33,7 @@ class FingerprintConfig:

@dataclass(slots=True)
 class GentleModeConfig:
+    # Мягкий режим загрузки страницы и capture.
    enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
@@ -43,6 +45,7 @@ class GentleModeConfig:

@dataclass(slots=True)
 class HumanPaceConfig:
+    # Паузы между действиями для более естественного поведения.
    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -60,6 +63,7 @@ class HumanPaceConfig:

@dataclass(slots=True)
 class ListingConfig:
+    # Ограничения и режим обхода листинга.
    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -70,26 +74,65 @@ class ListingConfig:

@dataclass(slots=True)
 class DatabaseConfig:
+    # Параметры подключения к БД.
    url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}


+@dataclass(slots=True)
+class ProxyConfig:
+    """Proxy settings for Playwright browser.
+
+    Supports HTTP, HTTPS and SOCKS5 proxies.
+    Format: ``protocol://[user:password@]host:port``
+
+    Examples:
+        - ``http://proxy.example.com:8080``
+        - ``socks5://user:pass@proxy.example.com:1080``
+    """
+    server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
+    username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
+    password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
@dataclass(slots=True)
 class Settings:
+    # Единая точка всех runtime-настроек приложения.
    home_url: str = "https://www.iaai.com/"
    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
    max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
+    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
+    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
+    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
+    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
    gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
    listing: ListingConfig = field(default_factory=ListingConfig)
    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)


+# Глобальные настройки по умолчанию.
 settings = Settings()
--- a/iaai_scraper/core/logs.py
+++ b/iaai_scraper/core/logs.py
@@ -1,14 +1,35 @@
 import logging
 import sys
+from contextvars import ContextVar
+
+
+# Trace id хранится в контексте текущей операции.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Подмешиваем trace_id в каждую log record.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    # Обновляем trace_id для текущего потока выполнения.
+    TRACE_ID.set(trace_id)


 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # Базовая настройка консольного и файлового логирования.
    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
    if log_file:
        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
    logging.basicConfig(
        level=getattr(logging, level.upper(), logging.INFO),
-        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
        handlers=handlers,
        force=True,
    )
--- a/iaai_scraper/core/retry.py
+++ b/iaai_scraper/core/retry.py
@@ -1,4 +1,5 @@
 import logging
+import random
 import time
 from collections.abc import Callable
 from functools import wraps
@@ -8,8 +9,23 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError

 logger = logging.getLogger("iaai_scraper.retry")

+# Ошибки, которые считаем временными и пригодными для повтора.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+)

-def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    # Универсальный retry с backoff и небольшим случайным jitter.
    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
        @wraps(func)
        def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -17,11 +33,15 @@ def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callab
            for attempt in range(1, max_attempts + 1):
                try:
                    return func(*args, **kwargs)
-                except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc:
+                except RETRYABLE_EXCEPTIONS as exc:
                    last_error = exc
                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
                    if attempt < max_attempts:
-                        time.sleep(delay_seconds * (2 ** (attempt - 1)))
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
            if last_error is not None:
                raise last_error
            raise RuntimeError("Retry wrapper failed without a captured exception")