cleanup and fix runtime bugs

iaai_scraper/scraper.py

@@ -12,6 +12,7 @@ from .browser import BrowserFactory, HumanPacer, NetworkCapture
 from .core.config import Settings, settings
 from .core.logs import set_trace_id, setup_logging
 from .core.retry import retryable
+from .core.runtime_config import RuntimeConfig
 from .core.utils import save_to_json
 from .parsing.mapper import CarMapper
 from .parsing.parser import VehicleParser
@@ -33,9 +34,9 @@ class IAAIScraper:
         return match.group(1)
 
     def __init__(self, runtime_settings: Settings | None = None) -> None:
-        # Базовые зависимости и сервисы скрапера.
         self.settings = runtime_settings or settings
         setup_logging(self.settings.log_level, self.settings.log_file)
+        self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file)
         self.trace_id = uuid.uuid4().hex[:12]
         set_trace_id(self.trace_id)
         self.playwright = None
@@ -49,10 +50,7 @@ class IAAIScraper:
         self.persistence = PersistenceService(self.settings)
         self._shutdown_requested = False
 
-    # browser lifecycle
-
     def _new_trace_id(self, prefix: str) -> str:
-        # Новый trace id для отдельной операции.
         trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
         self.trace_id = trace_id
         set_trace_id(trace_id)
@@ -69,7 +67,6 @@ class IAAIScraper:
         self.close()
 
     def close(self) -> None:
-        # Закрываем ресурсы аккуратно, даже если Playwright уже в ошибке.
         if self.context is not None:
             try:
                 self.context.close()
@@ -93,7 +90,6 @@ class IAAIScraper:
                 self.playwright = None
 
     def _new_context(self, storage_state: str | None = None) -> BrowserContext:
-        # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы.
         if self.browser is None:
             self.__enter__()
         if self.context:
@@ -109,10 +105,7 @@ class IAAIScraper:
         context = self._new_context()
         return context.new_page()
 
-    # listing
-
     def collect_listing(self, make: str | None = None, model: str | None = None):
-        # Собираем ссылки карточек с публичного листинга.
         page = self._get_unauthenticated_page()
 
         try:
@@ -130,43 +123,9 @@ class IAAIScraper:
             self._new_context()
         return self.context.new_page()
 
-    # scrape
-
-    @retryable(max_attempts=3, jitter_seconds=0.25)
-    def open_vehicle_page(self, vehicle_url: str):
-        # Лёгкое открытие страницы без сетевого дампа и сохранения в БД.
-        trace_id = self._new_trace_id("open")
-        started_at = time.perf_counter()
-        page = self._get_page()
-        try:
-            self.pacer.before_vehicle_open()
-            page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000)
-            try:
-                page.wait_for_load_state("networkidle", timeout=15000)
-            except PlaywrightTimeoutError:
-                pass
-            self.pacer.after_vehicle_open()
-
-            html = page.content()
-            try:
-                dom_text = page.locator("body").inner_text(timeout=10_000)
-            except Exception:
-                dom_text = ""
-            parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []})
-            return {
-                "trace_id": trace_id,
-                "source_url": vehicle_url,
-                "opened_in_light_mode": True,
-                "fetched_at_epoch": int(time.time()),
-                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
-                **parsed,
-            }
-        finally:
-            page.close()
-
     @retryable(max_attempts=3, jitter_seconds=0.25)
     def scrape_vehicle_detail(self, vehicle_url: str):
-        """Открыть страницу, перехватить JSON, вернуть данные."""
+        # Открыть страницу, перехватить JSON, вернуть данные.
         page = self._get_page()
         try:
             return self._scrape_on_page(page, vehicle_url)
@@ -174,7 +133,6 @@ class IAAIScraper:
             page.close()
 
     def _scrape_on_page(self, page: Page, vehicle_url: str):
-        # Полный проход по карточке с network capture и маппингом в DB-модель.
         trace_id = self._new_trace_id("scrape")
         started_at = time.perf_counter()
         capture = NetworkCapture(self.settings)
@@ -215,11 +173,8 @@ class IAAIScraper:
             save_to_json(network_dump, self.settings.raw_output_json)
         return result
 
-    # db sync
-
     def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
-        """Scrape + upsert одного авто."""
-        # Один URL -> один scrape -> один upsert.
+        # Scrape + upsert одного авто.
         trace_id = self._new_trace_id("sync-vehicle")
         started_at = time.perf_counter()
         self.persistence.create_tables()
@@ -274,14 +229,23 @@ class IAAIScraper:
         limit: int | None = None,
         only_new: bool | None = None,
     ):
-        """Листинг + sync всех найденных машин."""
-        # Массовая синхронизация с общим run_id и сбором ошибок.
+        # Листинг + sync всех найденных машин.
+        # Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет).
+        rc = self.runtime_config.sync
+        if limit is None and rc.limit is not None:
+            limit = rc.limit
+        if only_new is None and rc.only_new is not None:
+            only_new = rc.only_new
+        if lane == "iaai_cars" and rc.lane is not None:
+            lane = rc.lane
+
         trace_id = self._new_trace_id("sync-listing")
         started_at = time.perf_counter()
         self.persistence.create_tables()
         run_id = self.persistence.start_sync_run(lane=lane)
         cars_upserted = 0
         cars_failed = 0
+        cars_filtered = 0
         images_upserted = 0
         total = 0
         skipped_existing = 0
@@ -327,6 +291,29 @@ class IAAIScraper:
                     if not db_record:
                         raise RuntimeError("Scrape result does not contain db_record")
                     record = CarRecord.model_validate(db_record)
+
+                    # Применяем фильтры из runtime_config (include/exclude/price/mileage/flags)
+                    if not self.runtime_config.filters.is_empty():
+                        filter_values = {
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                            "is_damaged": record.is_damaged,
+                        }
+                        if not self.runtime_config.filters.matches(filter_values):
+                            logger.info(
+                                "[%d/%d] Skipped by filter: %s %s %s",
+                                index, total, record.brand, record.model, record.year or "?",
+                            )
+                            cars_filtered += 1
+                            continue
+
                     upsert = self.persistence.upsert_car(record)
                     if upsert.get("action") != "skipped":
                         cars_upserted += 1
@@ -351,7 +338,6 @@ class IAAIScraper:
                 if index < total:
                     self.pacer.between_vehicles()
         except Exception as exc:
-            # collect_listing itself failed — count as total failure
             if not failures:
                 failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
             logger.error("sync_listing failed: %s", exc)
@@ -369,8 +355,8 @@ class IAAIScraper:
             )
 
         logger.info(
-            "Sync run #%d finished: %d/%d upserted, %d failed, %d images",
-            run_id, cars_upserted, total, cars_failed, images_upserted,
+            "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images",
+            run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted,
         )
         return {
             "trace_id": trace_id,
@@ -379,17 +365,16 @@ class IAAIScraper:
             "listing": listing,
             "cars_upserted": cars_upserted,
             "cars_failed": cars_failed,
+            "cars_filtered": cars_filtered,
             "images_upserted": images_upserted,
             "skipped_existing": skipped_existing,
             "elapsed_seconds": round(time.perf_counter() - started_at, 3),
             "failures": failures,
         }
 
-    # scheduler
-
     def run_scheduled(self) -> None:
-        # Простой бесконечный цикл без внешнего планировщика.
-        # Graceful shutdown по SIGINT/SIGTERM.
+        # NOTE: Зарезервировано для standalone-режима (без Celery beat).
+        # В текущей архитектуре планирование выполняется через Celery beat + worker/tasks.py.
         def _handle_shutdown(signum, frame):
             logger.info("Received signal %s, shutting down gracefully...", signum)
             self._shutdown_requested = True
@@ -405,10 +390,9 @@ class IAAIScraper:
         cycle = 0
         while not self._shutdown_requested:
             cycle += 1
-            logger.info("=== Scheduler cycle #%d starting ===", cycle)
+            logger.info("Scheduler cycle #%d starting", cycle)
             start = time.time()
             try:
-                # Сбрасываем контекст перед каждым циклом.
                 if self.context:
                     try:
                         self.context.close()
@@ -416,7 +400,6 @@ class IAAIScraper:
                         pass
                     self.context = None
 
-                # Проверяем, что browser/playwright живы; пересоздаём при необходимости.
                 if self.browser is None or self.playwright is None:
                     logger.info("Browser/Playwright not available, re-initializing...")
                     self.close()
@@ -433,12 +416,10 @@ class IAAIScraper:
             except Exception as exc:
                 elapsed = time.time() - start
                 logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc)
-                # Полный сброс при любой ошибке цикла — следующий цикл пересоздаст всё.
                 try:
                     self.close()
                 except Exception:
                     pass
-                # Пересоздаём browser для следующего цикла.
                 try:
                     self.__enter__()
                 except Exception as reinit_exc:
@@ -450,7 +431,6 @@ class IAAIScraper:
             sleep_time = max(0, interval - (time.time() - start))
             if sleep_time > 0:
                 logger.info("Sleeping %.0f seconds until next cycle...", sleep_time)
-                # Прерываемый sleep — проверяем shutdown каждые 5 секунд.
                 slept = 0.0
                 while slept < sleep_time and not self._shutdown_requested:
                     chunk = min(5.0, sleep_time - slept)