cleanup and fix runtime bugs

2026-04-08 22:54:16 +03:00
parent 90bd4756b3
commit 6e97991c68
21 changed files with 220 additions and 400 deletions
--- a/iaai_scraper/core/config.py
+++ b/iaai_scraper/core/config.py
@@ -1,6 +1,5 @@
 import os
 from dataclasses import dataclass, field
-from pathlib import Path

 from dotenv import load_dotenv

@@ -9,7 +8,7 @@ load_dotenv()

@dataclass(slots=True)
 class FingerprintConfig:
-    # Настройки браузерного отпечатка.
+    # Браузерный отпечаток.
    user_agent: str = (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,20 +31,16 @@ class FingerprintConfig:


@dataclass(slots=True)
-class GentleModeConfig:
-    # Мягкий режим загрузки страницы и capture.
-    enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
+class CaptureConfig:
+    # Параметры capture.
    capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
    max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
    max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
-    warm_scroll_rounds: int = int(os.getenv("IAAI_WARM_SCROLL_ROUNDS", "1"))
-    scroll_pause_ms: int = int(os.getenv("IAAI_SCROLL_PAUSE_MS", "900"))
-    post_open_idle_ms: int = int(os.getenv("IAAI_POST_OPEN_IDLE_MS", "3500"))


@dataclass(slots=True)
 class HumanPaceConfig:
-    # Паузы между действиями для более естественного поведения.
+    # Паузы между действиями.
    enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
    after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
    after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -63,7 +58,7 @@ class HumanPaceConfig:

@dataclass(slots=True)
 class ListingConfig:
-    # Ограничения и режим обхода листинга.
+    # Лимиты листинга.
    cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
    max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
    max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -74,32 +69,43 @@ class ListingConfig:

@dataclass(slots=True)
 class DatabaseConfig:
-    # Параметры подключения к БД.
-    url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
+    # Подключение к БД.
+    url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
    echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
+    pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5"))
+    max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10"))
+
+
+@dataclass(slots=True)
+class RedisConfig:
+    # Подключение к Redis.
+    url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0")
+
+
+@dataclass(slots=True)
+class CeleryConfig:
+    # Настройки Celery.
+    broker_url: str = os.getenv("CELERY_BROKER_URL", "") or ""
+    result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or ""
+    task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600"))
+    task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900"))
+    worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1"))
+    beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60"))


@dataclass(slots=True)
 class ProxyConfig:
-    """Proxy settings for Playwright browser.
-
-    Supports HTTP, HTTPS and SOCKS5 proxies.
-    Format: ``protocol://[user:password@]host:port``
-
-    Examples:
-        - ``http://proxy.example.com:8080``
-        - ``socks5://user:pass@proxy.example.com:1080``
-    """
-    server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
-    username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
-    password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
+    # Настройки прокси.
+    server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None
+    username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None
+    password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None

    @property
    def enabled(self) -> bool:
        return bool(self.server)

    def to_playwright_dict(self) -> dict[str, str] | None:
-        """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
+        # Формат для Playwright.
        if not self.server:
            return None
        result: dict[str, str] = {"server": self.server}
@@ -112,7 +118,7 @@ class ProxyConfig:

@dataclass(slots=True)
 class Settings:
-    # Единая точка всех runtime-настроек приложения.
+    # Общие настройки.
    home_url: str = "https://www.iaai.com/"
    default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
    network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
@@ -121,19 +127,21 @@ class Settings:
    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
    headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
    log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
    log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
-    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
    sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
+    raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
+    scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
    listing: ListingConfig = field(default_factory=ListingConfig)
    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
    proxy: ProxyConfig = field(default_factory=ProxyConfig)


-# Глобальные настройки по умолчанию.
+# Настройки по умолчанию.
 settings = Settings()