improve scraper runtime

iaai_scraper/core/config.py

@@ -9,6 +9,7 @@ load_dotenv()
 
 @dataclass(slots=True)
 class FingerprintConfig:
+    # Настройки браузерного отпечатка.
     user_agent: str = (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
         "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,6 +33,7 @@ class FingerprintConfig:
 
 @dataclass(slots=True)
 class GentleModeConfig:
+    # Мягкий режим загрузки страницы и capture.
     enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
     capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
     max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
@@ -43,6 +45,7 @@ class GentleModeConfig:
 
 @dataclass(slots=True)
 class HumanPaceConfig:
+    # Паузы между действиями для более естественного поведения.
     enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
     after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
     after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -60,6 +63,7 @@ class HumanPaceConfig:
 
 @dataclass(slots=True)
 class ListingConfig:
+    # Ограничения и режим обхода листинга.
     cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
     max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
     max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -70,26 +74,65 @@ class ListingConfig:
 
 @dataclass(slots=True)
 class DatabaseConfig:
+    # Параметры подключения к БД.
     url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
     echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
 
 
+@dataclass(slots=True)
+class ProxyConfig:
+    """Proxy settings for Playwright browser.
+
+    Supports HTTP, HTTPS and SOCKS5 proxies.
+    Format: ``protocol://[user:password@]host:port``
+
+    Examples:
+        - ``http://proxy.example.com:8080``
+        - ``socks5://user:pass@proxy.example.com:1080``
+    """
+    server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
+    username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
+    password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
 @dataclass(slots=True)
 class Settings:
+    # Единая точка всех runtime-настроек приложения.
     home_url: str = "https://www.iaai.com/"
     default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
     network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
     max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
+    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
+    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
+    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
     headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
     raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
     log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
     log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
+    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
     scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
     fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
     gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
 
 
+# Глобальные настройки по умолчанию.
 settings = Settings()