update sync filtering

encar_scraper/encar.py

@@ -13,6 +13,7 @@ from urllib.request import Request, urlopen
 import urllib3
 
 from .core.config import Settings
+from .core.runtime_config import FiltersConfig
 from .storage.db import PersistenceService
 from .storage.schemas import CarRecord, ImageRecord
 from .translations import (
@@ -132,6 +133,23 @@ class EncarFilters:
         return "(And." + "._." .join(parts) + ".)"
 
 
+@dataclass(slots=True, frozen=True)
+class RuntimeFilterSpec:
+    price_min: int | None = None
+    price_max: int | None = None
+    mileage_min: int | None = None
+    mileage_max: int | None = None
+    models: frozenset[str] = frozenset()
+    years: frozenset[int] = frozenset()
+    body_types: frozenset[str] = frozenset()
+    colors: frozenset[str] = frozenset()
+    drives: frozenset[str] = frozenset()
+    gearboxes: frozenset[str] = frozenset()
+    exclude_models: frozenset[str] = frozenset()
+    exclude_years: frozenset[int] = frozenset()
+    exclude_body_types: frozenset[str] = frozenset()
+
+
 @dataclass(slots=True)
 class EncarMapper:
     def map_to_car_record(self, vehicle_url: str, payload: dict[str, Any], probe_all_photos: bool = False) -> CarRecord:
@@ -502,6 +520,99 @@ class EncarScraper:
             )
         return self._batch_pool
 
+    @staticmethod
+    def _compile_runtime_filters(runtime_filters: FiltersConfig | None) -> RuntimeFilterSpec | None:
+        if runtime_filters is None:
+            return None
+
+        spec = RuntimeFilterSpec(
+            price_min=runtime_filters.price_min,
+            price_max=runtime_filters.price_max,
+            mileage_min=runtime_filters.mileage_min,
+            mileage_max=runtime_filters.mileage_max,
+            models=frozenset(m for m in runtime_filters.models if m),
+            years=frozenset(int(y) for y in runtime_filters.years if y is not None),
+            body_types=frozenset(b for b in runtime_filters.body_types if b),
+            colors=frozenset(c for c in runtime_filters.colors if c),
+            drives=frozenset(d for d in runtime_filters.drives if d),
+            gearboxes=frozenset(g for g in runtime_filters.gearboxes if g),
+            exclude_models=frozenset(m for m in runtime_filters.exclude_models if m),
+            exclude_years=frozenset(int(y) for y in runtime_filters.exclude_years if y is not None),
+            exclude_body_types=frozenset(b for b in runtime_filters.exclude_body_types if b),
+        )
+
+        if not any((
+            spec.price_min is not None,
+            spec.price_max is not None,
+            spec.mileage_min is not None,
+            spec.mileage_max is not None,
+            spec.models,
+            spec.years,
+            spec.body_types,
+            spec.colors,
+            spec.drives,
+            spec.gearboxes,
+            spec.exclude_models,
+            spec.exclude_years,
+            spec.exclude_body_types,
+        )):
+            return None
+
+        return spec
+
+    @staticmethod
+    def _matches_any_token(value: str, tokens: frozenset[str]) -> bool:
+        if not tokens:
+            return True
+        return any(token in value for token in tokens if token)
+
+    def _record_passes_runtime_filters(
+        self,
+        record: CarRecord,
+        runtime_filters: RuntimeFilterSpec | None,
+    ) -> bool:
+        if runtime_filters is None:
+            return True
+
+        model_lower = (record.model or "").lower()
+        body_type_lower = (record.body_type or "").lower()
+        color_lower = (record.color or "").lower()
+        drive_lower = (record.drive or "").lower()
+        gearbox_lower = (record.gearbox or "").lower()
+
+        if runtime_filters.price_min is not None and (record.price is None or record.price < runtime_filters.price_min):
+            return False
+        if runtime_filters.price_max is not None and (record.price is None or record.price > runtime_filters.price_max):
+            return False
+        if runtime_filters.mileage_min is not None and record.mileage < runtime_filters.mileage_min:
+            return False
+        if runtime_filters.mileage_max is not None and record.mileage > runtime_filters.mileage_max:
+            return False
+
+        if runtime_filters.models and not self._matches_any_token(model_lower, runtime_filters.models):
+            return False
+        if runtime_filters.exclude_models and self._matches_any_token(model_lower, runtime_filters.exclude_models):
+            return False
+
+        if runtime_filters.years and (record.year is None or record.year not in runtime_filters.years):
+            return False
+        if runtime_filters.exclude_years and record.year in runtime_filters.exclude_years:
+            return False
+
+        if runtime_filters.body_types and body_type_lower not in runtime_filters.body_types:
+            return False
+        if runtime_filters.exclude_body_types and body_type_lower in runtime_filters.exclude_body_types:
+            return False
+
+        if runtime_filters.colors and color_lower not in runtime_filters.colors:
+            return False
+        if runtime_filters.drives and drive_lower not in runtime_filters.drives:
+            return False
+        if runtime_filters.gearboxes and gearbox_lower not in runtime_filters.gearboxes:
+            return False
+
+        return True
+
     def collect_listing(
         self,
         limit: int | None = None,
@@ -745,6 +856,7 @@ class EncarScraper:
         redis_client: Any | None = None,
         allowed_brands: set[str] | None = None,
         excluded_brands: set[str] | None = None,
+        runtime_filters: FiltersConfig | None = None,
         probe_all_photos: bool = False,
     ) -> dict[str, Any]:
         """Полная синхронизация листинга Encar.
@@ -763,6 +875,7 @@ class EncarScraper:
         self.persistence.create_tables()
         filters = filters or EncarFilters()
         page_size = self.DEFAULT_PAGE_SIZE
+        compiled_runtime_filters = self._compile_runtime_filters(runtime_filters)
 
         # --- Генерация шардов ---
         if limit:
@@ -886,7 +999,9 @@ class EncarScraper:
                 if len(batch) >= batch_size:
                     s, f, fids = self._flush_batch(
                         batch, all_origin_ids, allowed_brands,
-                        excluded_brands, probe_all_photos=probe_all_photos,
+                        excluded_brands,
+                        runtime_filters=compiled_runtime_filters,
+                        probe_all_photos=probe_all_photos,
                     )
                     synced += s
                     failed += f
@@ -922,7 +1037,9 @@ class EncarScraper:
             if batch:
                 s, f, fids = self._flush_batch(
                     batch, all_origin_ids, allowed_brands,
-                    excluded_brands, probe_all_photos=probe_all_photos,
+                    excluded_brands,
+                    runtime_filters=compiled_runtime_filters,
+                    probe_all_photos=probe_all_photos,
                 )
                 synced += s
                 failed += f
@@ -1002,6 +1119,7 @@ class EncarScraper:
         all_origin_ids: set[str],
         allowed_brands: set[str] | None = None,
         excluded_brands: set[str] | None = None,
+        runtime_filters: RuntimeFilterSpec | None = None,
         probe_all_photos: bool = False,
     ) -> tuple[int, int, list[str]]:
         """Маппит и upsert'ит батч items в БД. Возвращает (synced, failed, failed_ids)."""
@@ -1011,6 +1129,7 @@ class EncarScraper:
         failed = 0
         failed_ids: list[str] = []
         skipped_brands = 0
+        skipped_runtime = 0
 
         # Фаза 1: маппинг без пробинга + фильтр брендов
         for item in items:
@@ -1026,6 +1145,9 @@ class EncarScraper:
                 if excluded_brands and brand_lower in excluded_brands:
                     skipped_brands += 1
                     continue
+                if not self._record_passes_runtime_filters(record, runtime_filters):
+                    skipped_runtime += 1
+                    continue
 
                 records.append(record)
                 probe_tasks.append((len(records) - 1, item))
@@ -1063,8 +1185,10 @@ class EncarScraper:
             synced = 0
 
         total_images = sum(len(r.images) for r in records)
-        logger.info("Batch stats: %d items → %d after brand filter (-%d), %d synced, %d total images",
+        logger.info(
-                    len(items), len(records), skipped_brands, synced, total_images)
+            "Batch stats: %d items → %d after runtime filter (-%d brands, -%d runtime), %d synced, %d total images",
+            len(items), len(records), skipped_brands, skipped_runtime, synced, total_images,
+        )
 
         return synced, failed, failed_ids
 

encar_scraper/worker/tasks.py

@@ -200,6 +200,7 @@ def encar_sync_listing_task(
             redis_client=redis_client,
             allowed_brands=allowed_brands,
             excluded_brands=excluded_brands,
+            runtime_filters=rc.filters,
             probe_all_photos=rc.sync.probe_all_photos,
         )