"""Runtime config: загрузка и применение runtime_config.json. Позволяет менять параметры sync и фильтры машин без перезапуска сервиса. Файл перечитывается при каждом запуске sync_listing. """ from __future__ import annotations import json import logging from dataclasses import dataclass, field from pathlib import Path from typing import Any logger = logging.getLogger("openlane_scraper.core.runtime_config") @dataclass(slots=True) class SyncConfig: name: str | None = None ids_initial_size: int | None = None ids_next_size: int | None = None ids_max_pages: int | None = None condition_check_enabled: bool | None = None lane: str = "openlane_marketplace" only_new: bool = False limit: int | None = None @dataclass(slots=True) class ListingConfig: make: str | None = None model: str | None = None @dataclass(slots=True) class RangeFilter: min: int | None = None max: int | None = None @dataclass(slots=True) class FlagFilters: damaged_only: bool | None = None run_and_drive: bool | None = None @dataclass(slots=True) class FieldFilters: brands: list[str] = field(default_factory=list) models: list[str] = field(default_factory=list) years: list[int] = field(default_factory=list) body_types: list[str] = field(default_factory=list) colors: list[str] = field(default_factory=list) drives: list[str] = field(default_factory=list) gearboxes: list[str] = field(default_factory=list) locations: list[str] = field(default_factory=list) @dataclass(slots=True) class Filters: price: RangeFilter = field(default_factory=RangeFilter) mileage: RangeFilter = field(default_factory=RangeFilter) flags: FlagFilters = field(default_factory=FlagFilters) include: FieldFilters = field(default_factory=FieldFilters) exclude: FieldFilters = field(default_factory=FieldFilters) # Legacy flat fields (backward compatibility). brands: list[str] = field(default_factory=list) models: list[str] = field(default_factory=list) years: list[int] = field(default_factory=list) body_types: list[str] = field(default_factory=list) colors: list[str] = field(default_factory=list) drives: list[str] = field(default_factory=list) gearboxes: list[str] = field(default_factory=list) locations: list[str] = field(default_factory=list) exclude_brands: list[str] = field(default_factory=list) exclude_models: list[str] = field(default_factory=list) exclude_years: list[int] = field(default_factory=list) exclude_body_types: list[str] = field(default_factory=list) exclude_colors: list[str] = field(default_factory=list) exclude_drives: list[str] = field(default_factory=list) exclude_gearboxes: list[str] = field(default_factory=list) exclude_locations: list[str] = field(default_factory=list) @dataclass(slots=True) class RuntimeConfig: sync: SyncConfig = field(default_factory=SyncConfig) listing: ListingConfig = field(default_factory=ListingConfig) filters: Filters = field(default_factory=Filters) def load_runtime_config(path: str | Path | None) -> RuntimeConfig: """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты.""" if not path: return RuntimeConfig() p = Path(path) if not p.exists(): logger.debug("runtime_config not found at %s — using defaults", p) return RuntimeConfig() try: raw = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError) as exc: logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc) return RuntimeConfig() cfg = RuntimeConfig() sync_raw = raw.get("sync") or {} if isinstance(sync_raw, dict): cfg.sync.name = _opt_str(sync_raw.get("name")) cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size")) cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size")) cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages")) cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled")) cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane only_new = _opt_bool(sync_raw.get("only_new")) if only_new is not None: cfg.sync.only_new = only_new cfg.sync.limit = _opt_int(sync_raw.get("limit")) listing_raw = raw.get("listing") or {} if isinstance(listing_raw, dict): cfg.listing.make = _opt_str(listing_raw.get("make")) cfg.listing.model = _opt_str(listing_raw.get("model")) filters_raw = raw.get("filters") or {} if isinstance(filters_raw, dict): cfg.filters.price = _parse_range(filters_raw.get("price")) cfg.filters.mileage = _parse_range(filters_raw.get("mileage")) cfg.filters.flags = _parse_flags(filters_raw.get("flags")) include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw exclude_payload = ( filters_raw.get("exclude") if isinstance(filters_raw.get("exclude"), dict) else { "brands": filters_raw.get("exclude_brands"), "models": filters_raw.get("exclude_models"), "years": filters_raw.get("exclude_years"), "body_types": filters_raw.get("exclude_body_types"), "colors": filters_raw.get("exclude_colors"), "drives": filters_raw.get("exclude_drives"), "gearboxes": filters_raw.get("exclude_gearboxes"), "locations": filters_raw.get("exclude_locations"), } ) cfg.filters.include = _parse_fields(include_payload) cfg.filters.exclude = _parse_fields(exclude_payload) # Backward-compatible flat aliases. cfg.filters.brands = list(cfg.filters.include.brands) cfg.filters.models = list(cfg.filters.include.models) cfg.filters.years = list(cfg.filters.include.years) cfg.filters.body_types = list(cfg.filters.include.body_types) cfg.filters.colors = list(cfg.filters.include.colors) cfg.filters.drives = list(cfg.filters.include.drives) cfg.filters.gearboxes = list(cfg.filters.include.gearboxes) cfg.filters.locations = list(cfg.filters.include.locations) cfg.filters.exclude_brands = list(cfg.filters.exclude.brands) cfg.filters.exclude_models = list(cfg.filters.exclude.models) cfg.filters.exclude_years = list(cfg.filters.exclude.years) cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types) cfg.filters.exclude_colors = list(cfg.filters.exclude.colors) cfg.filters.exclude_drives = list(cfg.filters.exclude.drives) cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes) cfg.filters.exclude_locations = list(cfg.filters.exclude.locations) logger.info( "Loaded runtime_config from %s: lane=%s, limit=%s", p, cfg.sync.lane, cfg.sync.limit, ) return cfg def apply_filters(records: list[dict], filters: Filters) -> list[dict]: """Применить runtime-фильтры к списку сырых записей из API.""" if not records: return records include = getattr(filters, "include", None) exclude = getattr(filters, "exclude", None) include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ())) include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ())) include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ())) include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ())) exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ())) exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ())) exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ())) exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ())) price_cfg = getattr(filters, "price", None) mileage_cfg = getattr(filters, "mileage", None) flags_cfg = getattr(filters, "flags", None) price_min = getattr(price_cfg, "min", None) price_max = getattr(price_cfg, "max", None) mileage_min = getattr(mileage_cfg, "min", None) mileage_max = getattr(mileage_cfg, "max", None) damaged_only = getattr(flags_cfg, "damaged_only", None) run_and_drive = getattr(flags_cfg, "run_and_drive", None) result = records if include_brands: include_set = {s.casefold() for s in include_brands} result = [r for r in result if _get_brand(r).casefold() in include_set] if exclude_brands: exclude_set = {s.casefold() for s in exclude_brands} result = [r for r in result if _get_brand(r).casefold() not in exclude_set] if include_models: include_set = {s.casefold() for s in include_models} result = [r for r in result if _get_model(r).casefold() in include_set] if exclude_models: exclude_set = {s.casefold() for s in exclude_models} result = [r for r in result if _get_model(r).casefold() not in exclude_set] if include_years: years_set = set(include_years) result = [r for r in result if _get_year(r) in years_set] if exclude_years: years_set = set(exclude_years) result = [r for r in result if _get_year(r) not in years_set] if include_body_types: include_set = {s.casefold() for s in include_body_types} result = [r for r in result if _get_body_type(r).casefold() in include_set] if exclude_body_types: exclude_set = {s.casefold() for s in exclude_body_types} result = [r for r in result if _get_body_type(r).casefold() not in exclude_set] if price_min is not None or price_max is not None: filtered: list[dict] = [] for r in result: price = _get_price(r) if price is None: filtered.append(r) continue if price_min is not None and price < price_min: continue if price_max is not None and price > price_max: continue filtered.append(r) result = filtered if mileage_min is not None or mileage_max is not None: filtered = [] for r in result: miles = _get_mileage(r) if miles is None: filtered.append(r) continue if mileage_min is not None and miles < mileage_min: continue if mileage_max is not None and miles > mileage_max: continue filtered.append(r) result = filtered if damaged_only is not None: result = [r for r in result if bool(r.get("is_damaged")) is damaged_only] if run_and_drive is not None: result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive] return result def _parse_range(payload: Any) -> RangeFilter: payload = payload if isinstance(payload, dict) else {} return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max"))) def _parse_flags(payload: Any) -> FlagFilters: payload = payload if isinstance(payload, dict) else {} return FlagFilters( damaged_only=_opt_bool(payload.get("damaged_only")), run_and_drive=_opt_bool(payload.get("run_and_drive")), ) def _parse_fields(payload: Any) -> FieldFilters: payload = payload if isinstance(payload, dict) else {} return FieldFilters( brands=_str_list(payload.get("brands")), models=_str_list(payload.get("models")), years=_int_list(payload.get("years")), body_types=_str_list(payload.get("body_types")), colors=_str_list(payload.get("colors")), drives=_str_list(payload.get("drives")), gearboxes=_str_list(payload.get("gearboxes")), locations=_str_list(payload.get("locations")), ) def _opt_int(value: Any) -> int | None: if value is None: return None try: return int(value) except (ValueError, TypeError): return None def _opt_bool(value: Any) -> bool | None: if value is None: return None if isinstance(value, bool): return value if isinstance(value, str): normalized = value.strip().casefold() if normalized in {"1", "true", "yes", "on"}: return True if normalized in {"0", "false", "no", "off"}: return False return None def _opt_str(value: Any) -> str | None: if value is None: return None s = str(value).strip() return s or None def _str_list(value: Any) -> list[str]: if not isinstance(value, list): return [] return [str(v).strip() for v in value if str(v).strip()] def _int_list(value: Any) -> list[int]: if not isinstance(value, list): return [] result: list[int] = [] for v in value: iv = _opt_int(v) if iv is not None: result.append(iv) return result def _get_brand(record: dict[str, Any]) -> str: return str(record.get("make") or record.get("brand") or "") def _get_model(record: dict[str, Any]) -> str: return str(record.get("model") or "") def _get_body_type(record: dict[str, Any]) -> str: return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "") def _get_year(record: dict[str, Any]) -> int | None: return _opt_int(record.get("year")) def _get_price(record: dict[str, Any]) -> int | None: for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"): value = _opt_int(record.get(key)) if value is not None: return value return None def _get_mileage(record: dict[str, Any]) -> int | None: return _opt_int(record.get("odometer") or record.get("mileage"))