add openlane scraper

openlane_scraper/core/runtime_config.py

@@ -0,0 +1,379 @@
+"""Runtime config: загрузка и применение runtime_config.json.
+
+Позволяет менять параметры sync и фильтры машин без перезапуска сервиса.
+Файл перечитывается при каждом запуске sync_listing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("openlane_scraper.core.runtime_config")
+
+
+@dataclass(slots=True)
+class SyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str = "openlane_marketplace"
+    only_new: bool = False
+    limit: int | None = None
+
+
+@dataclass(slots=True)
+class ListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+
+@dataclass(slots=True)
+class RangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+
+@dataclass(slots=True)
+class FlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+
+@dataclass(slots=True)
+class FieldFilters:
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class Filters:
+    price: RangeFilter = field(default_factory=RangeFilter)
+    mileage: RangeFilter = field(default_factory=RangeFilter)
+    flags: FlagFilters = field(default_factory=FlagFilters)
+
+    include: FieldFilters = field(default_factory=FieldFilters)
+    exclude: FieldFilters = field(default_factory=FieldFilters)
+
+    # Legacy flat fields (backward compatibility).
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+    exclude_brands: list[str] = field(default_factory=list)
+    exclude_models: list[str] = field(default_factory=list)
+    exclude_years: list[int] = field(default_factory=list)
+    exclude_body_types: list[str] = field(default_factory=list)
+    exclude_colors: list[str] = field(default_factory=list)
+    exclude_drives: list[str] = field(default_factory=list)
+    exclude_gearboxes: list[str] = field(default_factory=list)
+    exclude_locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: SyncConfig = field(default_factory=SyncConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    filters: Filters = field(default_factory=Filters)
+
+
+def load_runtime_config(path: str | Path | None) -> RuntimeConfig:
+    """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты."""
+    if not path:
+        return RuntimeConfig()
+
+    p = Path(path)
+    if not p.exists():
+        logger.debug("runtime_config not found at %s — using defaults", p)
+        return RuntimeConfig()
+
+    try:
+        raw = json.loads(p.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as exc:
+        logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc)
+        return RuntimeConfig()
+
+    cfg = RuntimeConfig()
+
+    sync_raw = raw.get("sync") or {}
+    if isinstance(sync_raw, dict):
+        cfg.sync.name = _opt_str(sync_raw.get("name"))
+        cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size"))
+        cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size"))
+        cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages"))
+        cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled"))
+        cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane
+        only_new = _opt_bool(sync_raw.get("only_new"))
+        if only_new is not None:
+            cfg.sync.only_new = only_new
+        cfg.sync.limit = _opt_int(sync_raw.get("limit"))
+
+    listing_raw = raw.get("listing") or {}
+    if isinstance(listing_raw, dict):
+        cfg.listing.make = _opt_str(listing_raw.get("make"))
+        cfg.listing.model = _opt_str(listing_raw.get("model"))
+
+    filters_raw = raw.get("filters") or {}
+    if isinstance(filters_raw, dict):
+        cfg.filters.price = _parse_range(filters_raw.get("price"))
+        cfg.filters.mileage = _parse_range(filters_raw.get("mileage"))
+        cfg.filters.flags = _parse_flags(filters_raw.get("flags"))
+
+        include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw
+        exclude_payload = (
+            filters_raw.get("exclude")
+            if isinstance(filters_raw.get("exclude"), dict)
+            else {
+                "brands": filters_raw.get("exclude_brands"),
+                "models": filters_raw.get("exclude_models"),
+                "years": filters_raw.get("exclude_years"),
+                "body_types": filters_raw.get("exclude_body_types"),
+                "colors": filters_raw.get("exclude_colors"),
+                "drives": filters_raw.get("exclude_drives"),
+                "gearboxes": filters_raw.get("exclude_gearboxes"),
+                "locations": filters_raw.get("exclude_locations"),
+            }
+        )
+
+        cfg.filters.include = _parse_fields(include_payload)
+        cfg.filters.exclude = _parse_fields(exclude_payload)
+
+        # Backward-compatible flat aliases.
+        cfg.filters.brands = list(cfg.filters.include.brands)
+        cfg.filters.models = list(cfg.filters.include.models)
+        cfg.filters.years = list(cfg.filters.include.years)
+        cfg.filters.body_types = list(cfg.filters.include.body_types)
+        cfg.filters.colors = list(cfg.filters.include.colors)
+        cfg.filters.drives = list(cfg.filters.include.drives)
+        cfg.filters.gearboxes = list(cfg.filters.include.gearboxes)
+        cfg.filters.locations = list(cfg.filters.include.locations)
+
+        cfg.filters.exclude_brands = list(cfg.filters.exclude.brands)
+        cfg.filters.exclude_models = list(cfg.filters.exclude.models)
+        cfg.filters.exclude_years = list(cfg.filters.exclude.years)
+        cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types)
+        cfg.filters.exclude_colors = list(cfg.filters.exclude.colors)
+        cfg.filters.exclude_drives = list(cfg.filters.exclude.drives)
+        cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes)
+        cfg.filters.exclude_locations = list(cfg.filters.exclude.locations)
+
+    logger.info(
+        "Loaded runtime_config from %s: lane=%s, limit=%s",
+        p,
+        cfg.sync.lane,
+        cfg.sync.limit,
+    )
+    return cfg
+
+
+def apply_filters(records: list[dict], filters: Filters) -> list[dict]:
+    """Применить runtime-фильтры к списку сырых записей из API."""
+    if not records:
+        return records
+
+    include = getattr(filters, "include", None)
+    exclude = getattr(filters, "exclude", None)
+
+    include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ()))
+    include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ()))
+    include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ()))
+    include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ()))
+
+    exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ()))
+    exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ()))
+    exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ()))
+    exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ()))
+
+    price_cfg = getattr(filters, "price", None)
+    mileage_cfg = getattr(filters, "mileage", None)
+    flags_cfg = getattr(filters, "flags", None)
+
+    price_min = getattr(price_cfg, "min", None)
+    price_max = getattr(price_cfg, "max", None)
+    mileage_min = getattr(mileage_cfg, "min", None)
+    mileage_max = getattr(mileage_cfg, "max", None)
+    damaged_only = getattr(flags_cfg, "damaged_only", None)
+    run_and_drive = getattr(flags_cfg, "run_and_drive", None)
+
+    result = records
+
+    if include_brands:
+        include_set = {s.casefold() for s in include_brands}
+        result = [r for r in result if _get_brand(r).casefold() in include_set]
+    if exclude_brands:
+        exclude_set = {s.casefold() for s in exclude_brands}
+        result = [r for r in result if _get_brand(r).casefold() not in exclude_set]
+
+    if include_models:
+        include_set = {s.casefold() for s in include_models}
+        result = [r for r in result if _get_model(r).casefold() in include_set]
+    if exclude_models:
+        exclude_set = {s.casefold() for s in exclude_models}
+        result = [r for r in result if _get_model(r).casefold() not in exclude_set]
+
+    if include_years:
+        years_set = set(include_years)
+        result = [r for r in result if _get_year(r) in years_set]
+    if exclude_years:
+        years_set = set(exclude_years)
+        result = [r for r in result if _get_year(r) not in years_set]
+
+    if include_body_types:
+        include_set = {s.casefold() for s in include_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() in include_set]
+    if exclude_body_types:
+        exclude_set = {s.casefold() for s in exclude_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() not in exclude_set]
+
+    if price_min is not None or price_max is not None:
+        filtered: list[dict] = []
+        for r in result:
+            price = _get_price(r)
+            if price is None:
+                filtered.append(r)
+                continue
+            if price_min is not None and price < price_min:
+                continue
+            if price_max is not None and price > price_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if mileage_min is not None or mileage_max is not None:
+        filtered = []
+        for r in result:
+            miles = _get_mileage(r)
+            if miles is None:
+                filtered.append(r)
+                continue
+            if mileage_min is not None and miles < mileage_min:
+                continue
+            if mileage_max is not None and miles > mileage_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if damaged_only is not None:
+        result = [r for r in result if bool(r.get("is_damaged")) is damaged_only]
+    if run_and_drive is not None:
+        result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive]
+
+    return result
+
+
+def _parse_range(payload: Any) -> RangeFilter:
+    payload = payload if isinstance(payload, dict) else {}
+    return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max")))
+
+
+def _parse_flags(payload: Any) -> FlagFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FlagFilters(
+        damaged_only=_opt_bool(payload.get("damaged_only")),
+        run_and_drive=_opt_bool(payload.get("run_and_drive")),
+    )
+
+
+def _parse_fields(payload: Any) -> FieldFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FieldFilters(
+        brands=_str_list(payload.get("brands")),
+        models=_str_list(payload.get("models")),
+        years=_int_list(payload.get("years")),
+        body_types=_str_list(payload.get("body_types")),
+        colors=_str_list(payload.get("colors")),
+        drives=_str_list(payload.get("drives")),
+        gearboxes=_str_list(payload.get("gearboxes")),
+        locations=_str_list(payload.get("locations")),
+    )
+
+
+def _opt_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _opt_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+def _opt_str(value: Any) -> str | None:
+    if value is None:
+        return None
+    s = str(value).strip()
+    return s or None
+
+
+def _str_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(v).strip() for v in value if str(v).strip()]
+
+
+def _int_list(value: Any) -> list[int]:
+    if not isinstance(value, list):
+        return []
+    result: list[int] = []
+    for v in value:
+        iv = _opt_int(v)
+        if iv is not None:
+            result.append(iv)
+    return result
+
+
+def _get_brand(record: dict[str, Any]) -> str:
+    return str(record.get("make") or record.get("brand") or "")
+
+
+def _get_model(record: dict[str, Any]) -> str:
+    return str(record.get("model") or "")
+
+
+def _get_body_type(record: dict[str, Any]) -> str:
+    return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "")
+
+
+def _get_year(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("year"))
+
+
+def _get_price(record: dict[str, Any]) -> int | None:
+    for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"):
+        value = _opt_int(record.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _get_mileage(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("odometer") or record.get("mileage"))