From a6769b553f3662ad94a9178e3641fb8ceee45bc3 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 17 Apr 2026 13:29:54 +0300 Subject: [PATCH] fix runtime filtering --- encar_scraper/encar.py | 132 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 4 deletions(-) diff --git a/encar_scraper/encar.py b/encar_scraper/encar.py index f58daf2..d1540ef 100644 --- a/encar_scraper/encar.py +++ b/encar_scraper/encar.py @@ -13,6 +13,7 @@ from urllib.request import Request, urlopen import urllib3 from .core.config import Settings +from .core.runtime_config import FiltersConfig from .storage.db import PersistenceService from .storage.schemas import CarRecord, ImageRecord from .translations import ( @@ -132,6 +133,23 @@ class EncarFilters: return "(And." + "._." .join(parts) + ".)" +@dataclass(slots=True, frozen=True) +class RuntimeFilterSpec: + price_min: int | None = None + price_max: int | None = None + mileage_min: int | None = None + mileage_max: int | None = None + models: frozenset[str] = frozenset() + years: frozenset[int] = frozenset() + body_types: frozenset[str] = frozenset() + colors: frozenset[str] = frozenset() + drives: frozenset[str] = frozenset() + gearboxes: frozenset[str] = frozenset() + exclude_models: frozenset[str] = frozenset() + exclude_years: frozenset[int] = frozenset() + exclude_body_types: frozenset[str] = frozenset() + + @dataclass(slots=True) class EncarMapper: def map_to_car_record(self, vehicle_url: str, payload: dict[str, Any], probe_all_photos: bool = False) -> CarRecord: @@ -502,6 +520,99 @@ class EncarScraper: ) return self._batch_pool + @staticmethod + def _compile_runtime_filters(runtime_filters: FiltersConfig | None) -> RuntimeFilterSpec | None: + if runtime_filters is None: + return None + + spec = RuntimeFilterSpec( + price_min=runtime_filters.price_min, + price_max=runtime_filters.price_max, + mileage_min=runtime_filters.mileage_min, + mileage_max=runtime_filters.mileage_max, + models=frozenset(m for m in runtime_filters.models if m), + years=frozenset(int(y) for y in runtime_filters.years if y is not None), + body_types=frozenset(b for b in runtime_filters.body_types if b), + colors=frozenset(c for c in runtime_filters.colors if c), + drives=frozenset(d for d in runtime_filters.drives if d), + gearboxes=frozenset(g for g in runtime_filters.gearboxes if g), + exclude_models=frozenset(m for m in runtime_filters.exclude_models if m), + exclude_years=frozenset(int(y) for y in runtime_filters.exclude_years if y is not None), + exclude_body_types=frozenset(b for b in runtime_filters.exclude_body_types if b), + ) + + if not any(( + spec.price_min is not None, + spec.price_max is not None, + spec.mileage_min is not None, + spec.mileage_max is not None, + spec.models, + spec.years, + spec.body_types, + spec.colors, + spec.drives, + spec.gearboxes, + spec.exclude_models, + spec.exclude_years, + spec.exclude_body_types, + )): + return None + + return spec + + @staticmethod + def _matches_any_token(value: str, tokens: frozenset[str]) -> bool: + if not tokens: + return True + return any(token in value for token in tokens if token) + + def _record_passes_runtime_filters( + self, + record: CarRecord, + runtime_filters: RuntimeFilterSpec | None, + ) -> bool: + if runtime_filters is None: + return True + + model_lower = (record.model or "").lower() + body_type_lower = (record.body_type or "").lower() + color_lower = (record.color or "").lower() + drive_lower = (record.drive or "").lower() + gearbox_lower = (record.gearbox or "").lower() + + if runtime_filters.price_min is not None and (record.price is None or record.price < runtime_filters.price_min): + return False + if runtime_filters.price_max is not None and (record.price is None or record.price > runtime_filters.price_max): + return False + if runtime_filters.mileage_min is not None and record.mileage < runtime_filters.mileage_min: + return False + if runtime_filters.mileage_max is not None and record.mileage > runtime_filters.mileage_max: + return False + + if runtime_filters.models and not self._matches_any_token(model_lower, runtime_filters.models): + return False + if runtime_filters.exclude_models and self._matches_any_token(model_lower, runtime_filters.exclude_models): + return False + + if runtime_filters.years and (record.year is None or record.year not in runtime_filters.years): + return False + if runtime_filters.exclude_years and record.year in runtime_filters.exclude_years: + return False + + if runtime_filters.body_types and body_type_lower not in runtime_filters.body_types: + return False + if runtime_filters.exclude_body_types and body_type_lower in runtime_filters.exclude_body_types: + return False + + if runtime_filters.colors and color_lower not in runtime_filters.colors: + return False + if runtime_filters.drives and drive_lower not in runtime_filters.drives: + return False + if runtime_filters.gearboxes and gearbox_lower not in runtime_filters.gearboxes: + return False + + return True + def collect_listing( self, limit: int | None = None, @@ -745,6 +856,7 @@ class EncarScraper: redis_client: Any | None = None, allowed_brands: set[str] | None = None, excluded_brands: set[str] | None = None, + runtime_filters: FiltersConfig | None = None, probe_all_photos: bool = False, ) -> dict[str, Any]: """Полная синхронизация листинга Encar. @@ -763,6 +875,7 @@ class EncarScraper: self.persistence.create_tables() filters = filters or EncarFilters() page_size = self.DEFAULT_PAGE_SIZE + compiled_runtime_filters = self._compile_runtime_filters(runtime_filters) # --- Генерация шардов --- if limit: @@ -886,7 +999,9 @@ class EncarScraper: if len(batch) >= batch_size: s, f, fids = self._flush_batch( batch, all_origin_ids, allowed_brands, - excluded_brands, probe_all_photos=probe_all_photos, + excluded_brands, + runtime_filters=compiled_runtime_filters, + probe_all_photos=probe_all_photos, ) synced += s failed += f @@ -922,7 +1037,9 @@ class EncarScraper: if batch: s, f, fids = self._flush_batch( batch, all_origin_ids, allowed_brands, - excluded_brands, probe_all_photos=probe_all_photos, + excluded_brands, + runtime_filters=compiled_runtime_filters, + probe_all_photos=probe_all_photos, ) synced += s failed += f @@ -1002,6 +1119,7 @@ class EncarScraper: all_origin_ids: set[str], allowed_brands: set[str] | None = None, excluded_brands: set[str] | None = None, + runtime_filters: RuntimeFilterSpec | None = None, probe_all_photos: bool = False, ) -> tuple[int, int, list[str]]: """Маппит и upsert'ит батч items в БД. Возвращает (synced, failed, failed_ids).""" @@ -1011,6 +1129,7 @@ class EncarScraper: failed = 0 failed_ids: list[str] = [] skipped_brands = 0 + skipped_runtime = 0 # Фаза 1: маппинг без пробинга + фильтр брендов for item in items: @@ -1026,6 +1145,9 @@ class EncarScraper: if excluded_brands and brand_lower in excluded_brands: skipped_brands += 1 continue + if not self._record_passes_runtime_filters(record, runtime_filters): + skipped_runtime += 1 + continue records.append(record) probe_tasks.append((len(records) - 1, item)) @@ -1063,8 +1185,10 @@ class EncarScraper: synced = 0 total_images = sum(len(r.images) for r in records) - logger.info("Batch stats: %d items → %d after brand filter (-%d), %d synced, %d total images", - len(items), len(records), skipped_brands, synced, total_images) + logger.info( + "Batch stats: %d items → %d after runtime filter (-%d brands, -%d runtime), %d synced, %d total images", + len(items), len(records), skipped_brands, skipped_runtime, synced, total_images, + ) return synced, failed, failed_ids