add openlane scraper

openlane_scraper/core/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

openlane_scraper/core/config.py

@@ -0,0 +1,194 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+# --- Конфиг OpenLane (auth, paging, throttle, retry, checkpoint/output) ---
+
+@dataclass(slots=True)
+class OpenLaneConfig:
+    sign_in_url: str = _env_str("OPENLANE_SIGN_IN_URL", "https://app.openlane.com/sign_in")
+    search_url: str = _env_str("OPENLANE_SEARCH_URL", "https://app.openlane.com/search?tab=marketplace")
+    api_search_url: str = _env_str("OPENLANE_API_SEARCH_URL", "https://app.openlane.com/api/v4/search")
+    username: str = _env_str("OPENLANE_USERNAME", "")
+    password: str = _env_str("OPENLANE_PASSWORD", "")
+    source_tab: str = _env_str("OPENLANE_SOURCE_TAB", "marketplace")
+    sale_types: str = _env_str("OPENLANE_SALE_TYPES", "marketplace")
+    page_size: int = _env_int("OPENLANE_PAGE_SIZE", 30)
+    max_pages: int = _env_int("OPENLANE_MAX_PAGES", 512)
+    concurrency: int = _env_int("OPENLANE_CONCURRENCY", 4)
+    throttle_min_seconds: float = _env_float("OPENLANE_THROTTLE_MIN_SECONDS", 0.3)
+    throttle_max_seconds: float = _env_float("OPENLANE_THROTTLE_MAX_SECONDS", 0.8)
+    retry_schedule_seconds: tuple[float, ...] = tuple(
+        float(part.strip())
+        for part in _env_str("OPENLANE_RETRY_SCHEDULE_SECONDS", "2,5,10").split(",")
+        if part.strip()
+    ) or (2.0, 5.0, 10.0)
+    checkpoint_every_pages: int = _env_int("OPENLANE_CHECKPOINT_EVERY_PAGES", 10)
+    output_dir: str = _env_str("OPENLANE_OUTPUT_DIR", "artifacts/openlane")
+    jsonl_output: str = _env_str("OPENLANE_JSONL_OUTPUT", "artifacts/openlane/openlane_search.jsonl")
+    aggregated_output: str = _env_str("OPENLANE_AGGREGATED_OUTPUT", "artifacts/openlane/openlane_search_aggregated.json")
+    checkpoint_file: str = _env_str("OPENLANE_CHECKPOINT_FILE", "artifacts/openlane/openlane_checkpoint.json")
+    storage_state_file: str = _env_str("OPENLANE_STORAGE_STATE_FILE", "artifacts/openlane/openlane_storage_state.json")
+    persist_storage_state: bool = _env_bool("OPENLANE_PERSIST_STORAGE_STATE", True)
+    progress_log_every_pages: int = _env_int("OPENLANE_PROGRESS_LOG_EVERY_PAGES", 10)
+    request_timeout_ms: int = _env_int("OPENLANE_REQUEST_TIMEOUT_MS", 45000)
+    max_cars_limit: int = _env_int("OPENLANE_MAX_CARS_LIMIT", 20)
+    refresh_token: str = _env_str("OPENLANE_REFRESH_TOKEN", "")
+    okta_client_id: str = _env_str("OPENLANE_OKTA_CLIENT_ID", "")
+    okta_token_endpoint: str = _env_str(
+        "OPENLANE_OKTA_TOKEN_ENDPOINT",
+        "https://okta.iam.karglobal.com/oauth2/default/v1/token",
+    )
+    okta_redirect_uri: str = _env_str("OPENLANE_OKTA_REDIRECT_URI", "https://app.openlane.com/sign_in")
+    okta_timeout_seconds: int = _env_int("OPENLANE_OKTA_TIMEOUT_SECONDS", 15)
+
+
+# --- Конфиг PostgreSQL (URL, пул соединений, pool_recycle) ---
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("OPENLANE_DATABASE_URL", "postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper")
+    echo: bool = _env_bool("OPENLANE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("OPENLANE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("OPENLANE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("OPENLANE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("OPENLANE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("OPENLANE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("OPENLANE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("OPENLANE_PROXY_SERVER")
+    username: str | None = _env_optional_str("OPENLANE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("OPENLANE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    default_timeout_ms: int = _env_int("OPENLANE_TIMEOUT_MS", 45000)
+    headless: bool = _env_bool("OPENLANE_HEADLESS", True)
+    browser_engine: str = _env_str("OPENLANE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("OPENLANE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("OPENLANE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("OPENLANE_ENABLE_TRACE_ID_LOGS", True)
+    runtime_config_file: str | None = _env_path_str("OPENLANE_RUNTIME_CONFIG_FILE")
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    openlane: OpenLaneConfig = field(default_factory=OpenLaneConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

openlane_scraper/core/logs.py

@@ -0,0 +1,39 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+# Храним trace_id текущего потока/корутины.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
+    )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)

openlane_scraper/core/runtime_config.py

@@ -0,0 +1,379 @@
+"""Runtime config: загрузка и применение runtime_config.json.
+
+Позволяет менять параметры sync и фильтры машин без перезапуска сервиса.
+Файл перечитывается при каждом запуске sync_listing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("openlane_scraper.core.runtime_config")
+
+
+@dataclass(slots=True)
+class SyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str = "openlane_marketplace"
+    only_new: bool = False
+    limit: int | None = None
+
+
+@dataclass(slots=True)
+class ListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+
+@dataclass(slots=True)
+class RangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+
+@dataclass(slots=True)
+class FlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+
+@dataclass(slots=True)
+class FieldFilters:
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class Filters:
+    price: RangeFilter = field(default_factory=RangeFilter)
+    mileage: RangeFilter = field(default_factory=RangeFilter)
+    flags: FlagFilters = field(default_factory=FlagFilters)
+
+    include: FieldFilters = field(default_factory=FieldFilters)
+    exclude: FieldFilters = field(default_factory=FieldFilters)
+
+    # Legacy flat fields (backward compatibility).
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+    exclude_brands: list[str] = field(default_factory=list)
+    exclude_models: list[str] = field(default_factory=list)
+    exclude_years: list[int] = field(default_factory=list)
+    exclude_body_types: list[str] = field(default_factory=list)
+    exclude_colors: list[str] = field(default_factory=list)
+    exclude_drives: list[str] = field(default_factory=list)
+    exclude_gearboxes: list[str] = field(default_factory=list)
+    exclude_locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: SyncConfig = field(default_factory=SyncConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    filters: Filters = field(default_factory=Filters)
+
+
+def load_runtime_config(path: str | Path | None) -> RuntimeConfig:
+    """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты."""
+    if not path:
+        return RuntimeConfig()
+
+    p = Path(path)
+    if not p.exists():
+        logger.debug("runtime_config not found at %s — using defaults", p)
+        return RuntimeConfig()
+
+    try:
+        raw = json.loads(p.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as exc:
+        logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc)
+        return RuntimeConfig()
+
+    cfg = RuntimeConfig()
+
+    sync_raw = raw.get("sync") or {}
+    if isinstance(sync_raw, dict):
+        cfg.sync.name = _opt_str(sync_raw.get("name"))
+        cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size"))
+        cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size"))
+        cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages"))
+        cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled"))
+        cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane
+        only_new = _opt_bool(sync_raw.get("only_new"))
+        if only_new is not None:
+            cfg.sync.only_new = only_new
+        cfg.sync.limit = _opt_int(sync_raw.get("limit"))
+
+    listing_raw = raw.get("listing") or {}
+    if isinstance(listing_raw, dict):
+        cfg.listing.make = _opt_str(listing_raw.get("make"))
+        cfg.listing.model = _opt_str(listing_raw.get("model"))
+
+    filters_raw = raw.get("filters") or {}
+    if isinstance(filters_raw, dict):
+        cfg.filters.price = _parse_range(filters_raw.get("price"))
+        cfg.filters.mileage = _parse_range(filters_raw.get("mileage"))
+        cfg.filters.flags = _parse_flags(filters_raw.get("flags"))
+
+        include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw
+        exclude_payload = (
+            filters_raw.get("exclude")
+            if isinstance(filters_raw.get("exclude"), dict)
+            else {
+                "brands": filters_raw.get("exclude_brands"),
+                "models": filters_raw.get("exclude_models"),
+                "years": filters_raw.get("exclude_years"),
+                "body_types": filters_raw.get("exclude_body_types"),
+                "colors": filters_raw.get("exclude_colors"),
+                "drives": filters_raw.get("exclude_drives"),
+                "gearboxes": filters_raw.get("exclude_gearboxes"),
+                "locations": filters_raw.get("exclude_locations"),
+            }
+        )
+
+        cfg.filters.include = _parse_fields(include_payload)
+        cfg.filters.exclude = _parse_fields(exclude_payload)
+
+        # Backward-compatible flat aliases.
+        cfg.filters.brands = list(cfg.filters.include.brands)
+        cfg.filters.models = list(cfg.filters.include.models)
+        cfg.filters.years = list(cfg.filters.include.years)
+        cfg.filters.body_types = list(cfg.filters.include.body_types)
+        cfg.filters.colors = list(cfg.filters.include.colors)
+        cfg.filters.drives = list(cfg.filters.include.drives)
+        cfg.filters.gearboxes = list(cfg.filters.include.gearboxes)
+        cfg.filters.locations = list(cfg.filters.include.locations)
+
+        cfg.filters.exclude_brands = list(cfg.filters.exclude.brands)
+        cfg.filters.exclude_models = list(cfg.filters.exclude.models)
+        cfg.filters.exclude_years = list(cfg.filters.exclude.years)
+        cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types)
+        cfg.filters.exclude_colors = list(cfg.filters.exclude.colors)
+        cfg.filters.exclude_drives = list(cfg.filters.exclude.drives)
+        cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes)
+        cfg.filters.exclude_locations = list(cfg.filters.exclude.locations)
+
+    logger.info(
+        "Loaded runtime_config from %s: lane=%s, limit=%s",
+        p,
+        cfg.sync.lane,
+        cfg.sync.limit,
+    )
+    return cfg
+
+
+def apply_filters(records: list[dict], filters: Filters) -> list[dict]:
+    """Применить runtime-фильтры к списку сырых записей из API."""
+    if not records:
+        return records
+
+    include = getattr(filters, "include", None)
+    exclude = getattr(filters, "exclude", None)
+
+    include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ()))
+    include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ()))
+    include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ()))
+    include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ()))
+
+    exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ()))
+    exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ()))
+    exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ()))
+    exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ()))
+
+    price_cfg = getattr(filters, "price", None)
+    mileage_cfg = getattr(filters, "mileage", None)
+    flags_cfg = getattr(filters, "flags", None)
+
+    price_min = getattr(price_cfg, "min", None)
+    price_max = getattr(price_cfg, "max", None)
+    mileage_min = getattr(mileage_cfg, "min", None)
+    mileage_max = getattr(mileage_cfg, "max", None)
+    damaged_only = getattr(flags_cfg, "damaged_only", None)
+    run_and_drive = getattr(flags_cfg, "run_and_drive", None)
+
+    result = records
+
+    if include_brands:
+        include_set = {s.casefold() for s in include_brands}
+        result = [r for r in result if _get_brand(r).casefold() in include_set]
+    if exclude_brands:
+        exclude_set = {s.casefold() for s in exclude_brands}
+        result = [r for r in result if _get_brand(r).casefold() not in exclude_set]
+
+    if include_models:
+        include_set = {s.casefold() for s in include_models}
+        result = [r for r in result if _get_model(r).casefold() in include_set]
+    if exclude_models:
+        exclude_set = {s.casefold() for s in exclude_models}
+        result = [r for r in result if _get_model(r).casefold() not in exclude_set]
+
+    if include_years:
+        years_set = set(include_years)
+        result = [r for r in result if _get_year(r) in years_set]
+    if exclude_years:
+        years_set = set(exclude_years)
+        result = [r for r in result if _get_year(r) not in years_set]
+
+    if include_body_types:
+        include_set = {s.casefold() for s in include_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() in include_set]
+    if exclude_body_types:
+        exclude_set = {s.casefold() for s in exclude_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() not in exclude_set]
+
+    if price_min is not None or price_max is not None:
+        filtered: list[dict] = []
+        for r in result:
+            price = _get_price(r)
+            if price is None:
+                filtered.append(r)
+                continue
+            if price_min is not None and price < price_min:
+                continue
+            if price_max is not None and price > price_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if mileage_min is not None or mileage_max is not None:
+        filtered = []
+        for r in result:
+            miles = _get_mileage(r)
+            if miles is None:
+                filtered.append(r)
+                continue
+            if mileage_min is not None and miles < mileage_min:
+                continue
+            if mileage_max is not None and miles > mileage_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if damaged_only is not None:
+        result = [r for r in result if bool(r.get("is_damaged")) is damaged_only]
+    if run_and_drive is not None:
+        result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive]
+
+    return result
+
+
+def _parse_range(payload: Any) -> RangeFilter:
+    payload = payload if isinstance(payload, dict) else {}
+    return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max")))
+
+
+def _parse_flags(payload: Any) -> FlagFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FlagFilters(
+        damaged_only=_opt_bool(payload.get("damaged_only")),
+        run_and_drive=_opt_bool(payload.get("run_and_drive")),
+    )
+
+
+def _parse_fields(payload: Any) -> FieldFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FieldFilters(
+        brands=_str_list(payload.get("brands")),
+        models=_str_list(payload.get("models")),
+        years=_int_list(payload.get("years")),
+        body_types=_str_list(payload.get("body_types")),
+        colors=_str_list(payload.get("colors")),
+        drives=_str_list(payload.get("drives")),
+        gearboxes=_str_list(payload.get("gearboxes")),
+        locations=_str_list(payload.get("locations")),
+    )
+
+
+def _opt_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _opt_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+def _opt_str(value: Any) -> str | None:
+    if value is None:
+        return None
+    s = str(value).strip()
+    return s or None
+
+
+def _str_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(v).strip() for v in value if str(v).strip()]
+
+
+def _int_list(value: Any) -> list[int]:
+    if not isinstance(value, list):
+        return []
+    result: list[int] = []
+    for v in value:
+        iv = _opt_int(v)
+        if iv is not None:
+            result.append(iv)
+    return result
+
+
+def _get_brand(record: dict[str, Any]) -> str:
+    return str(record.get("make") or record.get("brand") or "")
+
+
+def _get_model(record: dict[str, Any]) -> str:
+    return str(record.get("model") or "")
+
+
+def _get_body_type(record: dict[str, Any]) -> str:
+    return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "")
+
+
+def _get_year(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("year"))
+
+
+def _get_price(record: dict[str, Any]) -> int | None:
+    for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"):
+        value = _opt_int(record.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _get_mileage(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("odometer") or record.get("mileage"))

openlane_scraper/core/utils.py

@@ -0,0 +1,66 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result