add mobilede scraper

2026-04-29 19:13:06 +03:00
parent b0bcd16ea2
commit f00819a21c
58 changed files with 15743 additions and 71 deletions
--- a/mobilede_scraper/core/config.py
+++ b/mobilede_scraper/core/config.py
@@ -0,0 +1,434 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import quote, urlsplit, urlunsplit
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("MOBILEDE_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("MOBILEDE_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("MOBILEDE_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("MOBILEDE_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.MOBILEDE.com/Vehiclelisting/Cars")
+    filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
+    filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
+    max_pages_per_run: int = _env_int("MOBILEDE_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("MOBILEDE_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("MOBILEDE_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("MOBILEDE_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("MOBILEDE_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("MOBILEDE_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации MOBILEDE (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
+    # или "runtime" для построения по runtime_config.filters.brands.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("MOBILEDE_LISTING_SEGMENTS", "")
+    fast_segment_year_splits: bool = _env_bool("MOBILEDE_FAST_SEGMENT_YEAR_SPLITS", True)
+
+    @property
+    def filtered_search_urls(self) -> list[str]:
+        urls: list[str] = []
+        if self.filtered_search_url and self.filtered_search_url.strip():
+            urls.append(self.filtered_search_url.strip())
+        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
+            raw = self.filtered_search_urls_raw.strip()
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                candidates = [str(item or "").strip() for item in parsed]
+            else:
+                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
+            for candidate in candidates:
+                if candidate and candidate not in urls:
+                    urls.append(candidate)
+        return urls
+
+
+# Список брендов MOBILEDE для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+MOBILEDE_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок mobilede: ~226 страниц × 100 = 22 600 результатов.
+MOBILEDE_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит сегменты по переданному списку брендов.
+
+    Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        if make in _LARGE_MAKES:
+            for yr_min, yr_max in _YEAR_SPLITS:
+                segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
+        else:
+            segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит HTTP-first сегменты без UI-фильтров годов.
+
+    Это ближе к MOBILEDE-fast: один бренд = один hidden-payload HTTP обход.
+    Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит MOBILEDE_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из MOBILEDE_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        return build_listing_segments_for_makes(list(MOBILEDE_DEFAULT_MAKES))
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper")
+    echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("MOBILEDE_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("MOBILEDE_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("MOBILEDE_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("MOBILEDE_ALWAYS_FULL_SCAN", False)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
+    fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
+
+
+@dataclass(slots=True)
+class ScrapingProfileConfig:
+    name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
+    http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
+    browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
+    anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
+    verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
+    verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
+    challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
+    challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
+    listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
+    request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
+    detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
+    listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
+
+    def __post_init__(self) -> None:
+        if self.name == "fast":
+            if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
+                self.browser_fallback_enabled = False
+            if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
+                self.anonymous_bootstrap_enabled = False
+            if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
+                self.challenge_refresh_attempts = 1
+            if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
+                self.listing_post_attempts = 2
+            if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
+                self.request_jitter_max_s = 0.0
+            if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
+                self.detail_retries = 1
+            if "MOBILEDE_LISTING_RETRIES" not in os.environ:
+                self.listing_retries = 1
+        else:
+            if self.detail_retries is not None and self.detail_retries < 0:
+                self.detail_retries = None
+            if self.listing_retries is not None and self.listing_retries < 0:
+                self.listing_retries = None
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER")
+    username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+    def to_requests_proxy_url(self) -> str | None:
+        if not self.server:
+            return None
+        if not self.username:
+            return self.server
+
+        parts = urlsplit(self.server)
+        if not parts.scheme or not parts.hostname:
+            return self.server
+
+        username = quote(self.username, safe="")
+        password = quote(self.password or "", safe="")
+        host = parts.hostname
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        if parts.port is not None:
+            host = f"{host}:{parts.port}"
+        netloc = f"{username}:{password}@{host}"
+        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+    def to_requests_proxies(self) -> dict[str, str] | None:
+        proxy_url = self.to_requests_proxy_url()
+        if not proxy_url:
+            return None
+        return {"http": proxy_url, "https": proxy_url}
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.MOBILEDE.com/"
+    default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
+    fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
+    browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def fetch_concurrency(self) -> int:
+        return self.celery.fetch_concurrency
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()