380 lines
14 KiB
Python
380 lines
14 KiB
Python
"""Runtime config: загрузка и применение runtime_config.json.
|
|
|
|
Позволяет менять параметры sync и фильтры машин без перезапуска сервиса.
|
|
Файл перечитывается при каждом запуске sync_listing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger("openlane_scraper.core.runtime_config")
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SyncConfig:
|
|
name: str | None = None
|
|
ids_initial_size: int | None = None
|
|
ids_next_size: int | None = None
|
|
ids_max_pages: int | None = None
|
|
condition_check_enabled: bool | None = None
|
|
lane: str = "openlane_marketplace"
|
|
only_new: bool = False
|
|
limit: int | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingConfig:
|
|
make: str | None = None
|
|
model: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RangeFilter:
|
|
min: int | None = None
|
|
max: int | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FlagFilters:
|
|
damaged_only: bool | None = None
|
|
run_and_drive: bool | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FieldFilters:
|
|
brands: list[str] = field(default_factory=list)
|
|
models: list[str] = field(default_factory=list)
|
|
years: list[int] = field(default_factory=list)
|
|
body_types: list[str] = field(default_factory=list)
|
|
colors: list[str] = field(default_factory=list)
|
|
drives: list[str] = field(default_factory=list)
|
|
gearboxes: list[str] = field(default_factory=list)
|
|
locations: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Filters:
|
|
price: RangeFilter = field(default_factory=RangeFilter)
|
|
mileage: RangeFilter = field(default_factory=RangeFilter)
|
|
flags: FlagFilters = field(default_factory=FlagFilters)
|
|
|
|
include: FieldFilters = field(default_factory=FieldFilters)
|
|
exclude: FieldFilters = field(default_factory=FieldFilters)
|
|
|
|
# Legacy flat fields (backward compatibility).
|
|
brands: list[str] = field(default_factory=list)
|
|
models: list[str] = field(default_factory=list)
|
|
years: list[int] = field(default_factory=list)
|
|
body_types: list[str] = field(default_factory=list)
|
|
colors: list[str] = field(default_factory=list)
|
|
drives: list[str] = field(default_factory=list)
|
|
gearboxes: list[str] = field(default_factory=list)
|
|
locations: list[str] = field(default_factory=list)
|
|
|
|
exclude_brands: list[str] = field(default_factory=list)
|
|
exclude_models: list[str] = field(default_factory=list)
|
|
exclude_years: list[int] = field(default_factory=list)
|
|
exclude_body_types: list[str] = field(default_factory=list)
|
|
exclude_colors: list[str] = field(default_factory=list)
|
|
exclude_drives: list[str] = field(default_factory=list)
|
|
exclude_gearboxes: list[str] = field(default_factory=list)
|
|
exclude_locations: list[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RuntimeConfig:
|
|
sync: SyncConfig = field(default_factory=SyncConfig)
|
|
listing: ListingConfig = field(default_factory=ListingConfig)
|
|
filters: Filters = field(default_factory=Filters)
|
|
|
|
|
|
def load_runtime_config(path: str | Path | None) -> RuntimeConfig:
|
|
"""Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты."""
|
|
if not path:
|
|
return RuntimeConfig()
|
|
|
|
p = Path(path)
|
|
if not p.exists():
|
|
logger.debug("runtime_config not found at %s — using defaults", p)
|
|
return RuntimeConfig()
|
|
|
|
try:
|
|
raw = json.loads(p.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError) as exc:
|
|
logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc)
|
|
return RuntimeConfig()
|
|
|
|
cfg = RuntimeConfig()
|
|
|
|
sync_raw = raw.get("sync") or {}
|
|
if isinstance(sync_raw, dict):
|
|
cfg.sync.name = _opt_str(sync_raw.get("name"))
|
|
cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size"))
|
|
cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size"))
|
|
cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages"))
|
|
cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled"))
|
|
cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane
|
|
only_new = _opt_bool(sync_raw.get("only_new"))
|
|
if only_new is not None:
|
|
cfg.sync.only_new = only_new
|
|
cfg.sync.limit = _opt_int(sync_raw.get("limit"))
|
|
|
|
listing_raw = raw.get("listing") or {}
|
|
if isinstance(listing_raw, dict):
|
|
cfg.listing.make = _opt_str(listing_raw.get("make"))
|
|
cfg.listing.model = _opt_str(listing_raw.get("model"))
|
|
|
|
filters_raw = raw.get("filters") or {}
|
|
if isinstance(filters_raw, dict):
|
|
cfg.filters.price = _parse_range(filters_raw.get("price"))
|
|
cfg.filters.mileage = _parse_range(filters_raw.get("mileage"))
|
|
cfg.filters.flags = _parse_flags(filters_raw.get("flags"))
|
|
|
|
include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw
|
|
exclude_payload = (
|
|
filters_raw.get("exclude")
|
|
if isinstance(filters_raw.get("exclude"), dict)
|
|
else {
|
|
"brands": filters_raw.get("exclude_brands"),
|
|
"models": filters_raw.get("exclude_models"),
|
|
"years": filters_raw.get("exclude_years"),
|
|
"body_types": filters_raw.get("exclude_body_types"),
|
|
"colors": filters_raw.get("exclude_colors"),
|
|
"drives": filters_raw.get("exclude_drives"),
|
|
"gearboxes": filters_raw.get("exclude_gearboxes"),
|
|
"locations": filters_raw.get("exclude_locations"),
|
|
}
|
|
)
|
|
|
|
cfg.filters.include = _parse_fields(include_payload)
|
|
cfg.filters.exclude = _parse_fields(exclude_payload)
|
|
|
|
# Backward-compatible flat aliases.
|
|
cfg.filters.brands = list(cfg.filters.include.brands)
|
|
cfg.filters.models = list(cfg.filters.include.models)
|
|
cfg.filters.years = list(cfg.filters.include.years)
|
|
cfg.filters.body_types = list(cfg.filters.include.body_types)
|
|
cfg.filters.colors = list(cfg.filters.include.colors)
|
|
cfg.filters.drives = list(cfg.filters.include.drives)
|
|
cfg.filters.gearboxes = list(cfg.filters.include.gearboxes)
|
|
cfg.filters.locations = list(cfg.filters.include.locations)
|
|
|
|
cfg.filters.exclude_brands = list(cfg.filters.exclude.brands)
|
|
cfg.filters.exclude_models = list(cfg.filters.exclude.models)
|
|
cfg.filters.exclude_years = list(cfg.filters.exclude.years)
|
|
cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types)
|
|
cfg.filters.exclude_colors = list(cfg.filters.exclude.colors)
|
|
cfg.filters.exclude_drives = list(cfg.filters.exclude.drives)
|
|
cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes)
|
|
cfg.filters.exclude_locations = list(cfg.filters.exclude.locations)
|
|
|
|
logger.info(
|
|
"Loaded runtime_config from %s: lane=%s, limit=%s",
|
|
p,
|
|
cfg.sync.lane,
|
|
cfg.sync.limit,
|
|
)
|
|
return cfg
|
|
|
|
|
|
def apply_filters(records: list[dict], filters: Filters) -> list[dict]:
|
|
"""Применить runtime-фильтры к списку сырых записей из API."""
|
|
if not records:
|
|
return records
|
|
|
|
include = getattr(filters, "include", None)
|
|
exclude = getattr(filters, "exclude", None)
|
|
|
|
include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ()))
|
|
include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ()))
|
|
include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ()))
|
|
include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ()))
|
|
|
|
exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ()))
|
|
exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ()))
|
|
exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ()))
|
|
exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ()))
|
|
|
|
price_cfg = getattr(filters, "price", None)
|
|
mileage_cfg = getattr(filters, "mileage", None)
|
|
flags_cfg = getattr(filters, "flags", None)
|
|
|
|
price_min = getattr(price_cfg, "min", None)
|
|
price_max = getattr(price_cfg, "max", None)
|
|
mileage_min = getattr(mileage_cfg, "min", None)
|
|
mileage_max = getattr(mileage_cfg, "max", None)
|
|
damaged_only = getattr(flags_cfg, "damaged_only", None)
|
|
run_and_drive = getattr(flags_cfg, "run_and_drive", None)
|
|
|
|
result = records
|
|
|
|
if include_brands:
|
|
include_set = {s.casefold() for s in include_brands}
|
|
result = [r for r in result if _get_brand(r).casefold() in include_set]
|
|
if exclude_brands:
|
|
exclude_set = {s.casefold() for s in exclude_brands}
|
|
result = [r for r in result if _get_brand(r).casefold() not in exclude_set]
|
|
|
|
if include_models:
|
|
include_set = {s.casefold() for s in include_models}
|
|
result = [r for r in result if _get_model(r).casefold() in include_set]
|
|
if exclude_models:
|
|
exclude_set = {s.casefold() for s in exclude_models}
|
|
result = [r for r in result if _get_model(r).casefold() not in exclude_set]
|
|
|
|
if include_years:
|
|
years_set = set(include_years)
|
|
result = [r for r in result if _get_year(r) in years_set]
|
|
if exclude_years:
|
|
years_set = set(exclude_years)
|
|
result = [r for r in result if _get_year(r) not in years_set]
|
|
|
|
if include_body_types:
|
|
include_set = {s.casefold() for s in include_body_types}
|
|
result = [r for r in result if _get_body_type(r).casefold() in include_set]
|
|
if exclude_body_types:
|
|
exclude_set = {s.casefold() for s in exclude_body_types}
|
|
result = [r for r in result if _get_body_type(r).casefold() not in exclude_set]
|
|
|
|
if price_min is not None or price_max is not None:
|
|
filtered: list[dict] = []
|
|
for r in result:
|
|
price = _get_price(r)
|
|
if price is None:
|
|
filtered.append(r)
|
|
continue
|
|
if price_min is not None and price < price_min:
|
|
continue
|
|
if price_max is not None and price > price_max:
|
|
continue
|
|
filtered.append(r)
|
|
result = filtered
|
|
|
|
if mileage_min is not None or mileage_max is not None:
|
|
filtered = []
|
|
for r in result:
|
|
miles = _get_mileage(r)
|
|
if miles is None:
|
|
filtered.append(r)
|
|
continue
|
|
if mileage_min is not None and miles < mileage_min:
|
|
continue
|
|
if mileage_max is not None and miles > mileage_max:
|
|
continue
|
|
filtered.append(r)
|
|
result = filtered
|
|
|
|
if damaged_only is not None:
|
|
result = [r for r in result if bool(r.get("is_damaged")) is damaged_only]
|
|
if run_and_drive is not None:
|
|
result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive]
|
|
|
|
return result
|
|
|
|
|
|
def _parse_range(payload: Any) -> RangeFilter:
|
|
payload = payload if isinstance(payload, dict) else {}
|
|
return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max")))
|
|
|
|
|
|
def _parse_flags(payload: Any) -> FlagFilters:
|
|
payload = payload if isinstance(payload, dict) else {}
|
|
return FlagFilters(
|
|
damaged_only=_opt_bool(payload.get("damaged_only")),
|
|
run_and_drive=_opt_bool(payload.get("run_and_drive")),
|
|
)
|
|
|
|
|
|
def _parse_fields(payload: Any) -> FieldFilters:
|
|
payload = payload if isinstance(payload, dict) else {}
|
|
return FieldFilters(
|
|
brands=_str_list(payload.get("brands")),
|
|
models=_str_list(payload.get("models")),
|
|
years=_int_list(payload.get("years")),
|
|
body_types=_str_list(payload.get("body_types")),
|
|
colors=_str_list(payload.get("colors")),
|
|
drives=_str_list(payload.get("drives")),
|
|
gearboxes=_str_list(payload.get("gearboxes")),
|
|
locations=_str_list(payload.get("locations")),
|
|
)
|
|
|
|
|
|
def _opt_int(value: Any) -> int | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return int(value)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def _opt_bool(value: Any) -> bool | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
normalized = value.strip().casefold()
|
|
if normalized in {"1", "true", "yes", "on"}:
|
|
return True
|
|
if normalized in {"0", "false", "no", "off"}:
|
|
return False
|
|
return None
|
|
|
|
|
|
def _opt_str(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
s = str(value).strip()
|
|
return s or None
|
|
|
|
|
|
def _str_list(value: Any) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(v).strip() for v in value if str(v).strip()]
|
|
|
|
|
|
def _int_list(value: Any) -> list[int]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
result: list[int] = []
|
|
for v in value:
|
|
iv = _opt_int(v)
|
|
if iv is not None:
|
|
result.append(iv)
|
|
return result
|
|
|
|
|
|
def _get_brand(record: dict[str, Any]) -> str:
|
|
return str(record.get("make") or record.get("brand") or "")
|
|
|
|
|
|
def _get_model(record: dict[str, Any]) -> str:
|
|
return str(record.get("model") or "")
|
|
|
|
|
|
def _get_body_type(record: dict[str, Any]) -> str:
|
|
return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "")
|
|
|
|
|
|
def _get_year(record: dict[str, Any]) -> int | None:
|
|
return _opt_int(record.get("year"))
|
|
|
|
|
|
def _get_price(record: dict[str, Any]) -> int | None:
|
|
for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"):
|
|
value = _opt_int(record.get(key))
|
|
if value is not None:
|
|
return value
|
|
return None
|
|
|
|
|
|
def _get_mileage(record: dict[str, Any]) -> int | None:
|
|
return _opt_int(record.get("odometer") or record.get("mileage"))
|