139 lines
6.6 KiB
Python
139 lines
6.6 KiB
Python
import os
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FingerprintConfig:
|
|
# Настройки браузерного отпечатка.
|
|
user_agent: str = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/135.0.0.0 Safari/537.36"
|
|
)
|
|
viewport_presets: tuple[dict[str, int], ...] = (
|
|
{"width": 1920, "height": 1080},
|
|
{"width": 1600, "height": 900},
|
|
{"width": 1536, "height": 864},
|
|
{"width": 1440, "height": 900},
|
|
{"width": 1366, "height": 768},
|
|
)
|
|
timezone_candidates: tuple[str, ...] = (
|
|
"America/New_York",
|
|
"America/Chicago",
|
|
"America/Los_Angeles",
|
|
)
|
|
locale: str = "en-US"
|
|
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class GentleModeConfig:
|
|
# Мягкий режим загрузки страницы и capture.
|
|
enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
|
|
max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
|
|
warm_scroll_rounds: int = int(os.getenv("IAAI_WARM_SCROLL_ROUNDS", "1"))
|
|
scroll_pause_ms: int = int(os.getenv("IAAI_SCROLL_PAUSE_MS", "900"))
|
|
post_open_idle_ms: int = int(os.getenv("IAAI_POST_OPEN_IDLE_MS", "3500"))
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class HumanPaceConfig:
|
|
# Паузы между действиями для более естественного поведения.
|
|
enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
|
|
after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
|
|
after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0"))
|
|
after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0"))
|
|
before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5"))
|
|
before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5"))
|
|
after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3"))
|
|
after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0"))
|
|
between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5"))
|
|
between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5"))
|
|
after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0"))
|
|
after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0"))
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingConfig:
|
|
# Ограничения и режим обхода листинга.
|
|
cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
|
|
max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
|
|
max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
|
|
page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200"))
|
|
include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class DatabaseConfig:
|
|
# Параметры подключения к БД.
|
|
url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
|
|
echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ProxyConfig:
|
|
"""Proxy settings for Playwright browser.
|
|
|
|
Supports HTTP, HTTPS and SOCKS5 proxies.
|
|
Format: ``protocol://[user:password@]host:port``
|
|
|
|
Examples:
|
|
- ``http://proxy.example.com:8080``
|
|
- ``socks5://user:pass@proxy.example.com:1080``
|
|
"""
|
|
server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
|
|
username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
|
|
password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return bool(self.server)
|
|
|
|
def to_playwright_dict(self) -> dict[str, str] | None:
|
|
"""Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
|
|
if not self.server:
|
|
return None
|
|
result: dict[str, str] = {"server": self.server}
|
|
if self.username:
|
|
result["username"] = self.username
|
|
if self.password:
|
|
result["password"] = self.password
|
|
return result
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Settings:
|
|
# Единая точка всех runtime-настроек приложения.
|
|
home_url: str = "https://www.iaai.com/"
|
|
default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
|
|
network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
|
|
max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
|
|
retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
|
|
retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
|
|
retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
|
|
headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
|
|
log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
|
|
log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
|
|
enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
|
scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
|
|
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
|
gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
|
|
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
|
listing: ListingConfig = field(default_factory=ListingConfig)
|
|
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
|
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
|
|
|
|
|
# Глобальные настройки по умолчанию.
|
|
settings = Settings()
|