import os from dataclasses import dataclass, field from dotenv import load_dotenv load_dotenv() @dataclass(slots=True) class FingerprintConfig: # Браузерный отпечаток. user_agent: str = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/135.0.0.0 Safari/537.36" ) viewport_presets: tuple[dict[str, int], ...] = ( {"width": 1920, "height": 1080}, {"width": 1600, "height": 900}, {"width": 1536, "height": 864}, {"width": 1440, "height": 900}, {"width": 1366, "height": 768}, ) timezone_candidates: tuple[str, ...] = ( "America/New_York", "America/Chicago", "America/Los_Angeles", ) locale: str = "en-US" sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' @dataclass(slots=True) class CaptureConfig: # Параметры capture. capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40")) max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30")) @dataclass(slots=True) class HumanPaceConfig: # Паузы между действиями. enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5")) after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5")) after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0")) after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0")) before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5")) before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5")) after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3")) after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0")) between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5")) between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5")) after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0")) after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0")) @dataclass(slots=True) class ListingConfig: # Лимиты листинга. cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5")) max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100")) page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200")) include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"} collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"} @dataclass(slots=True) class DatabaseConfig: # Подключение к БД. url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"} pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5")) max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10")) @dataclass(slots=True) class RedisConfig: # Подключение к Redis. url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0") @dataclass(slots=True) class CeleryConfig: # Настройки Celery. broker_url: str = os.getenv("CELERY_BROKER_URL", "") or "" result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or "" task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600")) task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900")) worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1")) beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60")) beat_sync_limit: int = int(os.getenv("CELERY_BEAT_SYNC_LIMIT", "26")) @dataclass(slots=True) class ProxyConfig: # Настройки прокси. server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None @property def enabled(self) -> bool: return bool(self.server) def to_playwright_dict(self) -> dict[str, str] | None: # Формат для Playwright. if not self.server: return None result: dict[str, str] = {"server": self.server} if self.username: result["username"] = self.username if self.password: result["password"] = self.password return result @dataclass(slots=True) class Settings: # Общие настройки. home_url: str = "https://www.iaai.com/" default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000")) network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800")) max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3")) retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5")) retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0")) retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25")) headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"} log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO") log_file: str | None = os.getenv("IAAI_LOG_FILE") or None enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"} sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"} raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60")) fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) capture: CaptureConfig = field(default_factory=CaptureConfig) pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) listing: ListingConfig = field(default_factory=ListingConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) # Настройки по умолчанию. settings = Settings()