improve scraper runtime

iaai_scraper/browser/factory.py

@@ -11,6 +11,7 @@ logger = logging.getLogger("iaai_scraper.browser")
 
 def _build_init_script() -> str:
     """JS-патч признаков автоматизации."""
+  # Подмешиваем базовые browser fingerprint overrides.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -61,11 +62,12 @@ def _build_init_script() -> str:
 class BrowserFactory:
 
     def __init__(self, settings: Settings) -> None:
+    # Фабрика браузера и контекстов на основе env-настроек.
         self.settings = settings
 
     def create_browser(self, playwright: Playwright) -> Browser:
         # пробуем Chrome, если нет — Chromium
-        launch_kwargs = {
+        launch_kwargs: dict = {
             "headless": self.settings.headless,
             "args": [
                 "--disable-blink-features=AutomationControlled",
@@ -74,6 +76,10 @@ class BrowserFactory:
                 "--disable-features=IsolateOrigins,site-per-process",
             ],
         }
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
         try:
             logger.info("Trying to launch real Chrome channel")
             return playwright.chromium.launch(channel="chrome", **launch_kwargs)
@@ -83,6 +89,7 @@ class BrowserFactory:
 
     def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
         # рандом viewport/timezone под каждый контекст
+      # Контекст изолирует cookies, headers и fingerprint-параметры.
         viewport = random.choice(self.settings.fingerprint.viewport_presets)
         timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
         color_scheme = random.choice(["light", "dark"])

iaai_scraper/browser/network.py

@@ -23,6 +23,7 @@ class NetworkCapture:
     _origin: str | None = None
 
     def attach(self, page: Page) -> None:
+        # Подписываемся на request/response события страницы.
         try:
             self._origin = urlparse(page.url).netloc.lower() or None
         except Exception:
@@ -31,6 +32,7 @@ class NetworkCapture:
         page.on("response", self._on_response)
 
     def _is_same_origin(self, url: str) -> bool:
+        # При необходимости ограничиваемся same-origin и доменами IAAI.
         if not self.settings.gentle.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
@@ -45,6 +47,7 @@ class NetworkCapture:
         if len(self.requests) >= self.settings.gentle.max_requests:
             return
         key = f"{request.method}:{request.url}:{request.post_data or ''}"
+	    # Дедупликация одинаковых запросов.
         if key in self._seen_req:
             return
         self._seen_req.add(key)
@@ -92,6 +95,7 @@ class NetworkCapture:
 
     @staticmethod
     def _categorize(url: str) -> str:
+	    # Простая эвристика для разбивки ответов по смыслу.
         low = url.lower()
         mapping = {
             "images": ["image", "media", "photos", "gallery"],

iaai_scraper/browser/pace.py

@@ -10,9 +10,11 @@ class HumanPacer:
     """Random паузы между действиями."""
 
     def __init__(self, settings: Settings) -> None:
+	    # Инкапсулирует все задержки между действиями браузера.
         self.settings = settings
 
     def pause(self, min_s: float, max_s: float) -> None:
+	    # Базовая случайная пауза в заданном диапазоне.
         if self.settings.pace.enabled:
             time.sleep(random.uniform(min_s, max_s))
 
@@ -35,6 +37,7 @@ class HumanPacer:
         self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
 
     def move_mouse_to(self, page: Page, locator: Locator) -> None:
+	    # Небольшое движение мыши к элементу перед кликом.
         try:
             box = locator.bounding_box()
         except Exception:

iaai_scraper/cli.py

@@ -6,7 +6,10 @@ from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
+    # Описание CLI-команд и их аргументов.
     parser = argparse.ArgumentParser(description="Public IAAI scraper")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
 
     init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
@@ -50,6 +53,7 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--make", default=None, help="Optional make filter")
     sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
     sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
+    sync_listing_parser.add_argument("--limit", type=int, default=None, help="Limit number of vehicles to sync")
     sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
 
     daemon_parser = subparsers.add_parser(
@@ -65,13 +69,24 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def main() -> None:
+    # Собираем runtime overrides только при необходимости.
     parser = build_parser()
     args = parser.parse_args()
 
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        from .core.config import Settings
+
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
     if args.command == "run-daemon":
         # daemon: бесконечный цикл
         from .core.config import Settings
-        runtime_settings = Settings()
+        runtime_settings = runtime_settings or Settings()
         if args.interval is not None:
             runtime_settings.scheduler_interval_minutes = args.interval
         with IAAIScraper(runtime_settings) as scraper:
@@ -80,7 +95,8 @@ def main() -> None:
         return
 
     # одноразовый запуск
-    with IAAIScraper() as scraper:
+    with IAAIScraper(runtime_settings) as scraper:
+        # Каждая команда сводится к одному методу скрапера.
         if args.command == "init-db":
             data = scraper.init_db()
         elif args.command == "collect-listing":
@@ -94,7 +110,7 @@ def main() -> None:
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
         else:
-            data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane)
+            data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane, limit=args.limit)
 
         save_to_json(data, Path(args.output))
         print(f"Saved result to {Path(args.output).resolve()}")

iaai_scraper/core/config.py

@@ -9,6 +9,7 @@ load_dotenv()
 
 @dataclass(slots=True)
 class FingerprintConfig:
+    # Настройки браузерного отпечатка.
     user_agent: str = (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
         "AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,6 +33,7 @@ class FingerprintConfig:
 
 @dataclass(slots=True)
 class GentleModeConfig:
+    # Мягкий режим загрузки страницы и capture.
     enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
     capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
     max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
@@ -43,6 +45,7 @@ class GentleModeConfig:
 
 @dataclass(slots=True)
 class HumanPaceConfig:
+    # Паузы между действиями для более естественного поведения.
     enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
     after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
     after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -60,6 +63,7 @@ class HumanPaceConfig:
 
 @dataclass(slots=True)
 class ListingConfig:
+    # Ограничения и режим обхода листинга.
     cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
     max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
     max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -70,26 +74,65 @@ class ListingConfig:
 
 @dataclass(slots=True)
 class DatabaseConfig:
+    # Параметры подключения к БД.
     url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
     echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
 
 
+@dataclass(slots=True)
+class ProxyConfig:
+    """Proxy settings for Playwright browser.
+
+    Supports HTTP, HTTPS and SOCKS5 proxies.
+    Format: ``protocol://[user:password@]host:port``
+
+    Examples:
+        - ``http://proxy.example.com:8080``
+        - ``socks5://user:pass@proxy.example.com:1080``
+    """
+    server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
+    username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
+    password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
 @dataclass(slots=True)
 class Settings:
+    # Единая точка всех runtime-настроек приложения.
     home_url: str = "https://www.iaai.com/"
     default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
     network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
     max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
+    retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
+    retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
+    retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
     headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
     raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
     log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
     log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
+    enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
     scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
     fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
     gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
 
 
+# Глобальные настройки по умолчанию.
 settings = Settings()

iaai_scraper/core/logs.py

@@ -1,14 +1,35 @@
 import logging
 import sys
+from contextvars import ContextVar
+
+
+# Trace id хранится в контексте текущей операции.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Подмешиваем trace_id в каждую log record.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    # Обновляем trace_id для текущего потока выполнения.
+    TRACE_ID.set(trace_id)
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # Базовая настройка консольного и файлового логирования.
     handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
     logging.basicConfig(
         level=getattr(logging, level.upper(), logging.INFO),
-        format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
         handlers=handlers,
         force=True,
     )

iaai_scraper/core/retry.py

@@ -1,4 +1,5 @@
 import logging
+import random
 import time
 from collections.abc import Callable
 from functools import wraps
@@ -8,8 +9,23 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
 logger = logging.getLogger("iaai_scraper.retry")
 
+# Ошибки, которые считаем временными и пригодными для повтора.
+RETRYABLE_EXCEPTIONS = (
+    PlaywrightTimeoutError,
+    Error,
+    ConnectionError,
+    OSError,
+    TimeoutError,
+)
 
-def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+
+def retryable(
+    max_attempts: int,
+    delay_seconds: float = 2.5,
+    backoff_multiplier: float = 2.0,
+    jitter_seconds: float = 0.0,
+) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    # Универсальный retry с backoff и небольшим случайным jitter.
     def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
         @wraps(func)
         def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -17,11 +33,15 @@ def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callab
             for attempt in range(1, max_attempts + 1):
                 try:
                     return func(*args, **kwargs)
-                except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc:
+                except RETRYABLE_EXCEPTIONS as exc:
                     last_error = exc
                     logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
                     if attempt < max_attempts:
-                        time.sleep(delay_seconds * (2 ** (attempt - 1)))
+                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
+                        if jitter_seconds > 0:
+                            sleep_for += random.uniform(0, jitter_seconds)
+                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
+                        time.sleep(sleep_for)
             if last_error is not None:
                 raise last_error
             raise RuntimeError("Retry wrapper failed without a captured exception")

iaai_scraper/scraper.py

@@ -1,5 +1,6 @@
 import logging
 import time
+import uuid
 from pathlib import Path
 from typing import Any
 
@@ -9,7 +10,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
 
 from .browser import BrowserFactory, HumanPacer, NetworkCapture
 from .core.config import Settings, settings
-from .core.logs import setup_logging
+from .core.logs import set_trace_id, setup_logging
 from .core.retry import retryable
 from .core.utils import save_to_json
 from .parsing.mapper import CarMapper
@@ -24,8 +25,11 @@ logger = logging.getLogger("iaai_scraper.scraper")
 class IAAIScraper:
 
     def __init__(self, runtime_settings: Settings | None = None) -> None:
+        # Базовые зависимости и сервисы скрапера.
         self.settings = runtime_settings or settings
         setup_logging(self.settings.log_level, self.settings.log_file)
+        self.trace_id = uuid.uuid4().hex[:12]
+        set_trace_id(self.trace_id)
         self.playwright = None
         self.browser = None
         self.context: BrowserContext | None = None
@@ -38,20 +42,51 @@ class IAAIScraper:
 
     # browser lifecycle
 
+    def _new_trace_id(self, prefix: str) -> str:
+        # Новый trace id для отдельной операции.
+        trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
+        self.trace_id = trace_id
+        set_trace_id(trace_id)
+        return trace_id
+
     def __enter__(self) -> "IAAIScraper":
-        self.playwright = sync_playwright().start()
+        if self.playwright is None:
-        self.browser = self.browser_factory.create_browser(self.playwright)
+            self.playwright = sync_playwright().start()
+        if self.browser is None:
+            self.browser = self.browser_factory.create_browser(self.playwright)
         return self
 
     def __exit__(self, exc_type, exc, tb) -> None:
-        if self.context:
+        self.close()
-            self.context.close()
+
-        if self.browser:
+    def close(self) -> None:
-            self.browser.close()
+        # Закрываем ресурсы аккуратно, даже если Playwright уже в ошибке.
-        if self.playwright:
+        if self.context is not None:
-            self.playwright.stop()
+            try:
+                self.context.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.context = None
+        if self.browser is not None:
+            try:
+                self.browser.close()
+            except PlaywrightError:
+                pass
+            finally:
+                self.browser = None
+        if self.playwright is not None:
+            try:
+                self.playwright.stop()
+            except PlaywrightError:
+                pass
+            finally:
+                self.playwright = None
 
     def _new_context(self, storage_state: str | None = None) -> BrowserContext:
+        # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы.
+        if self.browser is None:
+            self.__enter__()
         if self.context:
             self.context.close()
         self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state)
@@ -68,6 +103,7 @@ class IAAIScraper:
     # listing
 
     def collect_listing(self, make: str | None = None, model: str | None = None):
+        # Собираем ссылки карточек с публичного листинга.
         page = self._get_unauthenticated_page()
 
         try:
@@ -87,8 +123,11 @@ class IAAIScraper:
 
     # scrape
 
-    @retryable(max_attempts=3)
+    @retryable(max_attempts=3, jitter_seconds=0.25)
     def open_vehicle_page(self, vehicle_url: str):
+        # Лёгкое открытие страницы без сетевого дампа и сохранения в БД.
+        trace_id = self._new_trace_id("open")
+        started_at = time.perf_counter()
         page = self._get_page()
         try:
             self.pacer.before_vehicle_open()
@@ -107,15 +146,17 @@ class IAAIScraper:
                 dom_text = ""
             parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []})
             return {
+                "trace_id": trace_id,
                 "source_url": vehicle_url,
                 "opened_in_gentle_mode": True,
                 "fetched_at_epoch": int(time.time()),
+                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
                 **parsed,
             }
         finally:
             page.close()
 
-    @retryable(max_attempts=3)
+    @retryable(max_attempts=3, jitter_seconds=0.25)
     def scrape_vehicle_detail(self, vehicle_url: str):
         """Открыть страницу, перехватить JSON, вернуть данные."""
         page = self._get_page()
@@ -125,6 +166,9 @@ class IAAIScraper:
             page.close()
 
     def _scrape_on_page(self, page: Page, vehicle_url: str):
+        # Полный проход по карточке с network capture и маппингом в DB-модель.
+        trace_id = self._new_trace_id("scrape")
+        started_at = time.perf_counter()
         capture = NetworkCapture(self.settings)
         capture.attach(page)
 
@@ -150,8 +194,10 @@ class IAAIScraper:
         )
 
         result = {
+            "trace_id": trace_id,
             "source_url": vehicle_url,
             "fetched_at_epoch": int(time.time()),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
             "network": network_dump,
             **parsed,
             "db_record": db_record.model_dump(mode="json"),
@@ -165,6 +211,9 @@ class IAAIScraper:
 
     def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
         """Scrape + upsert одного авто."""
+        # Один URL -> один scrape -> один upsert.
+        trace_id = self._new_trace_id("sync-vehicle")
+        started_at = time.perf_counter()
         self.persistence.create_tables()
         run_id = self.persistence.start_sync_run(lane=lane)
         ids_fetched = 1
@@ -184,11 +233,13 @@ class IAAIScraper:
             images_upserted = int(upsert.get("images_upserted", 0))
             status = "success"
             return {
+                "trace_id": trace_id,
                 "status": status,
                 "run_id": run_id,
                 "vehicle_url": vehicle_url,
                 "db_action": upsert.get("action"),
                 "images_upserted": images_upserted,
+                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
                 "db_record": db_record,
             }
         except Exception as exc:
@@ -207,11 +258,16 @@ class IAAIScraper:
                 error_summary=error_summary,
             )
 
-    def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars"):
+    def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars", limit: int | None = None):
         """Листинг + sync всех найденных машин."""
+        # Массовая синхронизация с общим run_id и сбором ошибок.
+        trace_id = self._new_trace_id("sync-listing")
+        started_at = time.perf_counter()
         self.persistence.create_tables()
         listing = self.collect_listing(make=make, model=model)
         vehicle_urls = list(listing.get("vehicle_urls", []))
+        if limit is not None:
+            vehicle_urls = vehicle_urls[:max(0, limit)]
         run_id = self.persistence.start_sync_run(lane=lane)
         cars_upserted = 0
         cars_failed = 0
@@ -225,6 +281,7 @@ class IAAIScraper:
         try:
             for index, vehicle_url in enumerate(vehicle_urls, start=1):
                 try:
+                    # Повторно используем страницу, чтобы не создавать лишний overhead.
                     logger.info("[%d/%d] Scraping %s", index, total, vehicle_url)
                     scrape_result = self._scrape_on_page(page, vehicle_url)
                     db_record = scrape_result.get("db_record")
@@ -242,6 +299,7 @@ class IAAIScraper:
                         record.year or "?", record.price or "N/A", img_count,
                     )
                 except Exception as exc:
+                    # После сбоя пересоздаём page, чтобы не остаться в битом состоянии.
                     cars_failed += 1
                     failures.append({"vehicle_url": vehicle_url, "error": str(exc)})
                     logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc)
@@ -252,7 +310,7 @@ class IAAIScraper:
                     page = self._get_page()
 
                 if index < total:
-                    time.sleep(1.0)
+                    self.pacer.between_vehicles()
         finally:
             try:
                 page.close()
@@ -275,18 +333,21 @@ class IAAIScraper:
             run_id, cars_upserted, total, cars_failed, images_upserted,
         )
         return {
+            "trace_id": trace_id,
             "status": status,
             "run_id": run_id,
             "listing": listing,
             "cars_upserted": cars_upserted,
             "cars_failed": cars_failed,
             "images_upserted": images_upserted,
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
             "failures": failures,
         }
 
     # scheduler
 
     def run_scheduled(self) -> None:
+        # Простой бесконечный цикл без внешнего планировщика.
         interval = self.settings.scheduler_interval_minutes * 60
         logger.info(
             "Scheduler started: syncing every %d minutes",
@@ -332,6 +393,7 @@ class IAAIScraper:
 
     def _warm_page(self, page: Page) -> None:
         """Scroll для lazy-load."""
+        # Небольшой прогрев страницы для ленивых блоков и картинок.
         rounds = max(0, self.settings.gentle.warm_scroll_rounds)
         pause_seconds = max(0.1, self.settings.gentle.scroll_pause_ms / 1000)
         for _ in range(rounds):

iaai_scraper/storage/listing.py

@@ -15,6 +15,7 @@ VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
 
 @dataclass(slots=True)
 class ListingVehicleLink:
+	# Ссылка на одну карточку из листинга.
 	href: str
 	title: str = ""
 	lot_number: str | None = None
@@ -22,6 +23,7 @@ class ListingVehicleLink:
 
 @dataclass(slots=True)
 class ListingPageResult:
+	# Результат сбора ссылок с одной страницы листинга.
 	source_url: str
 	page_number: int
 	vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
@@ -31,10 +33,12 @@ class ListingPageResult:
 
 class ListingCollector:
 	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
+		# Сборщик ссылок из публичного листинга IAAI.
 		self.settings = settings
 		self.pacer = pacer
 
 	def open_cars_listing(self, page: Page) -> None:
+		# Открываем листинг и ждём появления ссылок на карточки.
 		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
 		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
 		try:
@@ -48,6 +52,7 @@ class ListingCollector:
 		self.pacer.after_listing_open()
 
 	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
+		# Пытаемся применить make/model фильтры через UI.
 		applied = {"make": None, "model": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
 			applied["make"] = make
@@ -58,6 +63,7 @@ class ListingCollector:
 		return applied
 
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
+		# Собираем ссылки только с текущей страницы.
 		anchors = page.locator("a[href*='/VehicleDetail/']")
 		total = min(anchors.count(), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
@@ -80,6 +86,7 @@ class ListingCollector:
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
 	def go_to_next_page(self, page: Page) -> bool:
+		# Переход на следующую страницу, если кнопка доступна.
 		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
 		for selector in selectors:
 			locator = page.locator(selector).first
@@ -101,6 +108,7 @@ class ListingCollector:
 		return False
 
 	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]:
+		# Последовательно собираем ссылки с учётом лимитов и пагинации.
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
 		pages: list[dict[str, object]] = []
@@ -121,6 +129,7 @@ class ListingCollector:
 
 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
+		# Пробуем несколько селекторов для одного и того же фильтра.
 		for selector in selectors:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
@@ -138,6 +147,7 @@ class ListingCollector:
 
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
+		# Грубая проверка наличия кнопки next.
 		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
 			if page.locator(selector).count() > 0:
 				return True