From f96e5ca5f8d538e40e812a4ccfe22965ce5dc8ff Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 8 Apr 2026 13:30:45 +0300 Subject: [PATCH] improve scraper runtime --- iaai_scraper/browser/factory.py | 9 +++- iaai_scraper/browser/network.py | 4 ++ iaai_scraper/browser/pace.py | 3 ++ iaai_scraper/cli.py | 22 +++++++-- iaai_scraper/core/config.py | 43 ++++++++++++++++ iaai_scraper/core/logs.py | 23 ++++++++- iaai_scraper/core/retry.py | 26 ++++++++-- iaai_scraper/scraper.py | 88 ++++++++++++++++++++++++++++----- iaai_scraper/storage/listing.py | 10 ++++ 9 files changed, 207 insertions(+), 21 deletions(-) diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py index 1a1f579..f96adbd 100644 --- a/iaai_scraper/browser/factory.py +++ b/iaai_scraper/browser/factory.py @@ -11,6 +11,7 @@ logger = logging.getLogger("iaai_scraper.browser") def _build_init_script() -> str: """JS-патч признаков автоматизации.""" + # Подмешиваем базовые browser fingerprint overrides. hardware_concurrency = random.choice([4, 8, 12, 16]) device_memory = random.choice([4, 8, 16]) languages = ["en-US", "en"] @@ -61,11 +62,12 @@ def _build_init_script() -> str: class BrowserFactory: def __init__(self, settings: Settings) -> None: + # Фабрика браузера и контекстов на основе env-настроек. self.settings = settings def create_browser(self, playwright: Playwright) -> Browser: # пробуем Chrome, если нет — Chromium - launch_kwargs = { + launch_kwargs: dict = { "headless": self.settings.headless, "args": [ "--disable-blink-features=AutomationControlled", @@ -74,6 +76,10 @@ class BrowserFactory: "--disable-features=IsolateOrigins,site-per-process", ], } + proxy_dict = self.settings.proxy.to_playwright_dict() + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) try: logger.info("Trying to launch real Chrome channel") return playwright.chromium.launch(channel="chrome", **launch_kwargs) @@ -83,6 +89,7 @@ class BrowserFactory: def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext: # рандом viewport/timezone под каждый контекст + # Контекст изолирует cookies, headers и fingerprint-параметры. viewport = random.choice(self.settings.fingerprint.viewport_presets) timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) color_scheme = random.choice(["light", "dark"]) diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py index dd5abf7..043f9f1 100644 --- a/iaai_scraper/browser/network.py +++ b/iaai_scraper/browser/network.py @@ -23,6 +23,7 @@ class NetworkCapture: _origin: str | None = None def attach(self, page: Page) -> None: + # Подписываемся на request/response события страницы. try: self._origin = urlparse(page.url).netloc.lower() or None except Exception: @@ -31,6 +32,7 @@ class NetworkCapture: page.on("response", self._on_response) def _is_same_origin(self, url: str) -> bool: + # При необходимости ограничиваемся same-origin и доменами IAAI. if not self.settings.gentle.capture_same_origin_only or not self._origin: return True netloc = urlparse(url).netloc.lower() @@ -45,6 +47,7 @@ class NetworkCapture: if len(self.requests) >= self.settings.gentle.max_requests: return key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Дедупликация одинаковых запросов. if key in self._seen_req: return self._seen_req.add(key) @@ -92,6 +95,7 @@ class NetworkCapture: @staticmethod def _categorize(url: str) -> str: + # Простая эвристика для разбивки ответов по смыслу. low = url.lower() mapping = { "images": ["image", "media", "photos", "gallery"], diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py index c141052..56ab605 100644 --- a/iaai_scraper/browser/pace.py +++ b/iaai_scraper/browser/pace.py @@ -10,9 +10,11 @@ class HumanPacer: """Random паузы между действиями.""" def __init__(self, settings: Settings) -> None: + # Инкапсулирует все задержки между действиями браузера. self.settings = settings def pause(self, min_s: float, max_s: float) -> None: + # Базовая случайная пауза в заданном диапазоне. if self.settings.pace.enabled: time.sleep(random.uniform(min_s, max_s)) @@ -35,6 +37,7 @@ class HumanPacer: self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) def move_mouse_to(self, page: Page, locator: Locator) -> None: + # Небольшое движение мыши к элементу перед кликом. try: box = locator.bounding_box() except Exception: diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py index b845028..ffaeec5 100644 --- a/iaai_scraper/cli.py +++ b/iaai_scraper/cli.py @@ -6,7 +6,10 @@ from .scraper import IAAIScraper def build_parser() -> argparse.ArgumentParser: + # Описание CLI-команд и их аргументов. parser = argparse.ArgumentParser(description="Public IAAI scraper") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") subparsers = parser.add_subparsers(dest="command", required=True) init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables") @@ -50,6 +53,7 @@ def build_parser() -> argparse.ArgumentParser: sync_listing_parser.add_argument("--make", default=None, help="Optional make filter") sync_listing_parser.add_argument("--model", default=None, help="Optional model filter") sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs") + sync_listing_parser.add_argument("--limit", type=int, default=None, help="Limit number of vehicles to sync") sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON") daemon_parser = subparsers.add_parser( @@ -65,13 +69,24 @@ def build_parser() -> argparse.ArgumentParser: def main() -> None: + # Собираем runtime overrides только при необходимости. parser = build_parser() args = parser.parse_args() + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + from .core.config import Settings + + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + if args.command == "run-daemon": # daemon: бесконечный цикл from .core.config import Settings - runtime_settings = Settings() + runtime_settings = runtime_settings or Settings() if args.interval is not None: runtime_settings.scheduler_interval_minutes = args.interval with IAAIScraper(runtime_settings) as scraper: @@ -80,7 +95,8 @@ def main() -> None: return # одноразовый запуск - with IAAIScraper() as scraper: + with IAAIScraper(runtime_settings) as scraper: + # Каждая команда сводится к одному методу скрапера. if args.command == "init-db": data = scraper.init_db() elif args.command == "collect-listing": @@ -94,7 +110,7 @@ def main() -> None: elif args.command == "sync-vehicle": data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) else: - data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane) + data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane, limit=args.limit) save_to_json(data, Path(args.output)) print(f"Saved result to {Path(args.output).resolve()}") diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index da2be73..1d1dd81 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -9,6 +9,7 @@ load_dotenv() @dataclass(slots=True) class FingerprintConfig: + # Настройки браузерного отпечатка. user_agent: str = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " @@ -32,6 +33,7 @@ class FingerprintConfig: @dataclass(slots=True) class GentleModeConfig: + # Мягкий режим загрузки страницы и capture. enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"} capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40")) @@ -43,6 +45,7 @@ class GentleModeConfig: @dataclass(slots=True) class HumanPaceConfig: + # Паузы между действиями для более естественного поведения. enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5")) after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5")) @@ -60,6 +63,7 @@ class HumanPaceConfig: @dataclass(slots=True) class ListingConfig: + # Ограничения и режим обхода листинга. cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5")) max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100")) @@ -70,26 +74,65 @@ class ListingConfig: @dataclass(slots=True) class DatabaseConfig: + # Параметры подключения к БД. url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db") echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"} +@dataclass(slots=True) +class ProxyConfig: + """Proxy settings for Playwright browser. + + Supports HTTP, HTTPS and SOCKS5 proxies. + Format: ``protocol://[user:password@]host:port`` + + Examples: + - ``http://proxy.example.com:8080`` + - ``socks5://user:pass@proxy.example.com:1080`` + """ + server: str | None = os.getenv("IAAI_PROXY_SERVER") or None + username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None + password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + """Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*.""" + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + @dataclass(slots=True) class Settings: + # Единая точка всех runtime-настроек приложения. home_url: str = "https://www.iaai.com/" default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000")) network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800")) max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3")) + retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5")) + retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0")) + retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25")) headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"} raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO") log_file: str | None = os.getenv("IAAI_LOG_FILE") or None + enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"} scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60")) fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) gentle: GentleModeConfig = field(default_factory=GentleModeConfig) pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) listing: ListingConfig = field(default_factory=ListingConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) +# Глобальные настройки по умолчанию. settings = Settings() diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py index a071546..53f8452 100644 --- a/iaai_scraper/core/logs.py +++ b/iaai_scraper/core/logs.py @@ -1,14 +1,35 @@ import logging import sys +from contextvars import ContextVar + + +# Trace id хранится в контексте текущей операции. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Подмешиваем trace_id в каждую log record. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + # Обновляем trace_id для текущего потока выполнения. + TRACE_ID.set(trace_id) def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # Базовая настройка консольного и файлового логирования. handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] if log_file: handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) logging.basicConfig( level=getattr(logging, level.upper(), logging.INFO), - format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s", handlers=handlers, force=True, ) diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py index b08fe6a..6ae8772 100644 --- a/iaai_scraper/core/retry.py +++ b/iaai_scraper/core/retry.py @@ -1,4 +1,5 @@ import logging +import random import time from collections.abc import Callable from functools import wraps @@ -8,8 +9,23 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError logger = logging.getLogger("iaai_scraper.retry") +# Ошибки, которые считаем временными и пригодными для повтора. +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, +) -def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + # Универсальный retry с backoff и небольшим случайным jitter. def decorator(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Any: @@ -17,11 +33,15 @@ def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callab for attempt in range(1, max_attempts + 1): try: return func(*args, **kwargs) - except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc: + except RETRYABLE_EXCEPTIONS as exc: last_error = exc logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) if attempt < max_attempts: - time.sleep(delay_seconds * (2 ** (attempt - 1))) + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) if last_error is not None: raise last_error raise RuntimeError("Retry wrapper failed without a captured exception") diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 812d55c..8e60caf 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -1,5 +1,6 @@ import logging import time +import uuid from pathlib import Path from typing import Any @@ -9,7 +10,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from .browser import BrowserFactory, HumanPacer, NetworkCapture from .core.config import Settings, settings -from .core.logs import setup_logging +from .core.logs import set_trace_id, setup_logging from .core.retry import retryable from .core.utils import save_to_json from .parsing.mapper import CarMapper @@ -24,8 +25,11 @@ logger = logging.getLogger("iaai_scraper.scraper") class IAAIScraper: def __init__(self, runtime_settings: Settings | None = None) -> None: + # Базовые зависимости и сервисы скрапера. self.settings = runtime_settings or settings setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) self.playwright = None self.browser = None self.context: BrowserContext | None = None @@ -38,20 +42,51 @@ class IAAIScraper: # browser lifecycle + def _new_trace_id(self, prefix: str) -> str: + # Новый trace id для отдельной операции. + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + def __enter__(self) -> "IAAIScraper": - self.playwright = sync_playwright().start() - self.browser = self.browser_factory.create_browser(self.playwright) + if self.playwright is None: + self.playwright = sync_playwright().start() + if self.browser is None: + self.browser = self.browser_factory.create_browser(self.playwright) return self def __exit__(self, exc_type, exc, tb) -> None: - if self.context: - self.context.close() - if self.browser: - self.browser.close() - if self.playwright: - self.playwright.stop() + self.close() + + def close(self) -> None: + # Закрываем ресурсы аккуратно, даже если Playwright уже в ошибке. + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None def _new_context(self, storage_state: str | None = None) -> BrowserContext: + # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы. + if self.browser is None: + self.__enter__() if self.context: self.context.close() self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state) @@ -68,6 +103,7 @@ class IAAIScraper: # listing def collect_listing(self, make: str | None = None, model: str | None = None): + # Собираем ссылки карточек с публичного листинга. page = self._get_unauthenticated_page() try: @@ -87,8 +123,11 @@ class IAAIScraper: # scrape - @retryable(max_attempts=3) + @retryable(max_attempts=3, jitter_seconds=0.25) def open_vehicle_page(self, vehicle_url: str): + # Лёгкое открытие страницы без сетевого дампа и сохранения в БД. + trace_id = self._new_trace_id("open") + started_at = time.perf_counter() page = self._get_page() try: self.pacer.before_vehicle_open() @@ -107,15 +146,17 @@ class IAAIScraper: dom_text = "" parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []}) return { + "trace_id": trace_id, "source_url": vehicle_url, "opened_in_gentle_mode": True, "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), **parsed, } finally: page.close() - @retryable(max_attempts=3) + @retryable(max_attempts=3, jitter_seconds=0.25) def scrape_vehicle_detail(self, vehicle_url: str): """Открыть страницу, перехватить JSON, вернуть данные.""" page = self._get_page() @@ -125,6 +166,9 @@ class IAAIScraper: page.close() def _scrape_on_page(self, page: Page, vehicle_url: str): + # Полный проход по карточке с network capture и маппингом в DB-модель. + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() capture = NetworkCapture(self.settings) capture.attach(page) @@ -150,8 +194,10 @@ class IAAIScraper: ) result = { + "trace_id": trace_id, "source_url": vehicle_url, "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), "network": network_dump, **parsed, "db_record": db_record.model_dump(mode="json"), @@ -165,6 +211,9 @@ class IAAIScraper: def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): """Scrape + upsert одного авто.""" + # Один URL -> один scrape -> один upsert. + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() self.persistence.create_tables() run_id = self.persistence.start_sync_run(lane=lane) ids_fetched = 1 @@ -184,11 +233,13 @@ class IAAIScraper: images_upserted = int(upsert.get("images_upserted", 0)) status = "success" return { + "trace_id": trace_id, "status": status, "run_id": run_id, "vehicle_url": vehicle_url, "db_action": upsert.get("action"), "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), "db_record": db_record, } except Exception as exc: @@ -207,11 +258,16 @@ class IAAIScraper: error_summary=error_summary, ) - def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars"): + def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars", limit: int | None = None): """Листинг + sync всех найденных машин.""" + # Массовая синхронизация с общим run_id и сбором ошибок. + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() self.persistence.create_tables() listing = self.collect_listing(make=make, model=model) vehicle_urls = list(listing.get("vehicle_urls", [])) + if limit is not None: + vehicle_urls = vehicle_urls[:max(0, limit)] run_id = self.persistence.start_sync_run(lane=lane) cars_upserted = 0 cars_failed = 0 @@ -225,6 +281,7 @@ class IAAIScraper: try: for index, vehicle_url in enumerate(vehicle_urls, start=1): try: + # Повторно используем страницу, чтобы не создавать лишний overhead. logger.info("[%d/%d] Scraping %s", index, total, vehicle_url) scrape_result = self._scrape_on_page(page, vehicle_url) db_record = scrape_result.get("db_record") @@ -242,6 +299,7 @@ class IAAIScraper: record.year or "?", record.price or "N/A", img_count, ) except Exception as exc: + # После сбоя пересоздаём page, чтобы не остаться в битом состоянии. cars_failed += 1 failures.append({"vehicle_url": vehicle_url, "error": str(exc)}) logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc) @@ -252,7 +310,7 @@ class IAAIScraper: page = self._get_page() if index < total: - time.sleep(1.0) + self.pacer.between_vehicles() finally: try: page.close() @@ -275,18 +333,21 @@ class IAAIScraper: run_id, cars_upserted, total, cars_failed, images_upserted, ) return { + "trace_id": trace_id, "status": status, "run_id": run_id, "listing": listing, "cars_upserted": cars_upserted, "cars_failed": cars_failed, "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), "failures": failures, } # scheduler def run_scheduled(self) -> None: + # Простой бесконечный цикл без внешнего планировщика. interval = self.settings.scheduler_interval_minutes * 60 logger.info( "Scheduler started: syncing every %d minutes", @@ -332,6 +393,7 @@ class IAAIScraper: def _warm_page(self, page: Page) -> None: """Scroll для lazy-load.""" + # Небольшой прогрев страницы для ленивых блоков и картинок. rounds = max(0, self.settings.gentle.warm_scroll_rounds) pause_seconds = max(0.1, self.settings.gentle.scroll_pause_ms / 1000) for _ in range(rounds): diff --git a/iaai_scraper/storage/listing.py b/iaai_scraper/storage/listing.py index ea8b712..bf11f15 100644 --- a/iaai_scraper/storage/listing.py +++ b/iaai_scraper/storage/listing.py @@ -15,6 +15,7 @@ VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE) @dataclass(slots=True) class ListingVehicleLink: + # Ссылка на одну карточку из листинга. href: str title: str = "" lot_number: str | None = None @@ -22,6 +23,7 @@ class ListingVehicleLink: @dataclass(slots=True) class ListingPageResult: + # Результат сбора ссылок с одной страницы листинга. source_url: str page_number: int vehicle_links: list[ListingVehicleLink] = field(default_factory=list) @@ -31,10 +33,12 @@ class ListingPageResult: class ListingCollector: def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + # Сборщик ссылок из публичного листинга IAAI. self.settings = settings self.pacer = pacer def open_cars_listing(self, page: Page) -> None: + # Открываем листинг и ждём появления ссылок на карточки. logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded") try: @@ -48,6 +52,7 @@ class ListingCollector: self.pacer.after_listing_open() def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: + # Пытаемся применить make/model фильтры через UI. applied = {"make": None, "model": None} if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): applied["make"] = make @@ -58,6 +63,7 @@ class ListingCollector: return applied def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Собираем ссылки только с текущей страницы. anchors = page.locator("a[href*='/VehicleDetail/']") total = min(anchors.count(), self.settings.listing.page_link_limit) links: list[ListingVehicleLink] = [] @@ -80,6 +86,7 @@ class ListingCollector: return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) def go_to_next_page(self, page: Page) -> bool: + # Переход на следующую страницу, если кнопка доступна. selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] for selector in selectors: locator = page.locator(selector).first @@ -101,6 +108,7 @@ class ListingCollector: return False def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]: + # Последовательно собираем ссылки с учётом лимитов и пагинации. self.open_cars_listing(page) applied_filters = self.apply_filters(page, make=make, model=model) pages: list[dict[str, object]] = [] @@ -121,6 +129,7 @@ class ListingCollector: @staticmethod def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + # Пробуем несколько селекторов для одного и того же фильтра. for selector in selectors: locator = page.locator(selector).first if locator.count() == 0: @@ -138,6 +147,7 @@ class ListingCollector: @staticmethod def _has_next_page(page: Page) -> bool: + # Грубая проверка наличия кнопки next. for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: if page.locator(selector).count() > 0: return True