qananasikq/Openlane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve scraper runtime

Browse Source
This commit is contained in:
qananasikq
2026-04-08 13:30:45 +03:00
parent 21bdf17f35
commit f96e5ca5f8
9 changed files with 207 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
iaai_scraper/browser/factory.py
View File

@@ -11,6 +11,7 @@ logger = logging.getLogger("iaai_scraper.browser")
def _build_init_script() -> str:
"""JS-патч признаков автоматизации."""
# Подмешиваем базовые browser fingerprint overrides.
hardware_concurrency = random.choice([4, 8, 12, 16])
device_memory = random.choice([4, 8, 16])
languages = ["en-US", "en"]
@@ -61,11 +62,12 @@ def _build_init_script() -> str:
class BrowserFactory:
def __init__(self, settings: Settings) -> None:
# Фабрика браузера и контекстов на основе env-настроек.
self.settings = settings
def create_browser(self, playwright: Playwright) -> Browser:
# пробуем Chrome, если нет — Chromium
launch_kwargs = {
launch_kwargs: dict = {
"headless": self.settings.headless,
"args": [
"--disable-blink-features=AutomationControlled",
@@ -74,6 +76,10 @@ class BrowserFactory:
"--disable-features=IsolateOrigins,site-per-process",
],
}
proxy_dict = self.settings.proxy.to_playwright_dict()
if proxy_dict:
launch_kwargs["proxy"] = proxy_dict
logger.info("Using proxy: %s", self.settings.proxy.server)
try:
logger.info("Trying to launch real Chrome channel")
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
@@ -83,6 +89,7 @@ class BrowserFactory:
def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
# рандом viewport/timezone под каждый контекст
# Контекст изолирует cookies, headers и fingerprint-параметры.
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])

4
iaai_scraper/browser/network.py
View File

@@ -23,6 +23,7 @@ class NetworkCapture:
_origin: str | None = None
def attach(self, page: Page) -> None:
# Подписываемся на request/response события страницы.
try:
self._origin = urlparse(page.url).netloc.lower() or None
except Exception:
@@ -31,6 +32,7 @@ class NetworkCapture:
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# При необходимости ограничиваемся same-origin и доменами IAAI.
if not self.settings.gentle.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
@@ -45,6 +47,7 @@ class NetworkCapture:
if len(self.requests) >= self.settings.gentle.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Дедупликация одинаковых запросов.
if key in self._seen_req:
return
self._seen_req.add(key)
@@ -92,6 +95,7 @@ class NetworkCapture:
@staticmethod
def _categorize(url: str) -> str:
# Простая эвристика для разбивки ответов по смыслу.
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],

3
iaai_scraper/browser/pace.py
View File

@@ -10,9 +10,11 @@ class HumanPacer:
"""Random паузы между действиями."""
def __init__(self, settings: Settings) -> None:
# Инкапсулирует все задержки между действиями браузера.
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
# Базовая случайная пауза в заданном диапазоне.
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
@@ -35,6 +37,7 @@ class HumanPacer:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
# Небольшое движение мыши к элементу перед кликом.
try:
box = locator.bounding_box()
except Exception:

22
iaai_scraper/cli.py
View File

@@ -6,7 +6,10 @@ from .scraper import IAAIScraper
def build_parser() -> argparse.ArgumentParser:
# Описание CLI-команд и их аргументов.
parser = argparse.ArgumentParser(description="Public IAAI scraper")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
@@ -50,6 +53,7 @@ def build_parser() -> argparse.ArgumentParser:
sync_listing_parser.add_argument("--make", default=None, help="Optional make filter")
sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
sync_listing_parser.add_argument("--limit", type=int, default=None, help="Limit number of vehicles to sync")
sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
daemon_parser = subparsers.add_parser(
@@ -65,13 +69,24 @@ def build_parser() -> argparse.ArgumentParser:
def main() -> None:
# Собираем runtime overrides только при необходимости.
parser = build_parser()
args = parser.parse_args()
runtime_settings: Settings | None = None
if args.headless is not None or args.debug:
from .core.config import Settings
runtime_settings = Settings()
if args.headless is not None:
runtime_settings.headless = args.headless == "true"
if args.debug:
runtime_settings.log_level = "DEBUG"
if args.command == "run-daemon":
# daemon: бесконечный цикл
from .core.config import Settings
runtime_settings = Settings()
runtime_settings = runtime_settings or Settings()
if args.interval is not None:
runtime_settings.scheduler_interval_minutes = args.interval
with IAAIScraper(runtime_settings) as scraper:
@@ -80,7 +95,8 @@ def main() -> None:
return
# одноразовый запуск
with IAAIScraper() as scraper:
with IAAIScraper(runtime_settings) as scraper:
# Каждая команда сводится к одному методу скрапера.
if args.command == "init-db":
data = scraper.init_db()
elif args.command == "collect-listing":
@@ -94,7 +110,7 @@ def main() -> None:
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane)
data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane, limit=args.limit)
save_to_json(data, Path(args.output))
print(f"Saved result to {Path(args.output).resolve()}")

43
iaai_scraper/core/config.py
View File

@@ -9,6 +9,7 @@ load_dotenv()
@dataclass(slots=True)
class FingerprintConfig:
# Настройки браузерного отпечатка.
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,6 +33,7 @@ class FingerprintConfig:
@dataclass(slots=True)
class GentleModeConfig:
# Мягкий режим загрузки страницы и capture.
enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
@@ -43,6 +45,7 @@ class GentleModeConfig:
@dataclass(slots=True)
class HumanPaceConfig:
# Паузы между действиями для более естественного поведения.
enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -60,6 +63,7 @@ class HumanPaceConfig:
@dataclass(slots=True)
class ListingConfig:
# Ограничения и режим обхода листинга.
cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -70,26 +74,65 @@ class ListingConfig:
@dataclass(slots=True)
class DatabaseConfig:
# Параметры подключения к БД.
url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
@dataclass(slots=True)
class ProxyConfig:
"""Proxy settings for Playwright browser.
Supports HTTP, HTTPS and SOCKS5 proxies.
Format: ``protocol://[user:password@]host:port``
Examples:
- ``http://proxy.example.com:8080``
- ``socks5://user:pass@proxy.example.com:1080``
"""
server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
"""Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
@dataclass(slots=True)
class Settings:
# Единая точка всех runtime-настроек приложения.
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3"))
retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5"))
retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
# Глобальные настройки по умолчанию.
settings = Settings()

23
iaai_scraper/core/logs.py
View File

@@ -1,14 +1,35 @@
import logging
import sys
from contextvars import ContextVar
# Trace id хранится в контексте текущей операции.
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
class TraceIdFilter(logging.Filter):
# Подмешиваем trace_id в каждую log record.
def filter(self, record: logging.LogRecord) -> bool:
record.trace_id = TRACE_ID.get()
return True
def set_trace_id(trace_id: str) -> None:
# Обновляем trace_id для текущего потока выполнения.
TRACE_ID.set(trace_id)
def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
# Базовая настройка консольного и файлового логирования.
handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
if log_file:
handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
trace_filter = TraceIdFilter()
for handler in handlers:
handler.addFilter(trace_filter)
logging.basicConfig(
level=getattr(logging, level.upper(), logging.INFO),
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
handlers=handlers,
force=True,
)

26
iaai_scraper/core/retry.py
View File

@@ -1,4 +1,5 @@
import logging
import random
import time
from collections.abc import Callable
from functools import wraps
@@ -8,8 +9,23 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
logger = logging.getLogger("iaai_scraper.retry")
# Ошибки, которые считаем временными и пригодными для повтора.
RETRYABLE_EXCEPTIONS = (
PlaywrightTimeoutError,
Error,
ConnectionError,
OSError,
TimeoutError,
)
def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
def retryable(
max_attempts: int,
delay_seconds: float = 2.5,
backoff_multiplier: float = 2.0,
jitter_seconds: float = 0.0,
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
# Универсальный retry с backoff и небольшим случайным jitter.
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -17,11 +33,15 @@ def retryable(max_attempts: int, delay_seconds: float = 2.5) -> Callable[[Callab
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except (PlaywrightTimeoutError, Error, ConnectionError, OSError, RuntimeError) as exc:
except RETRYABLE_EXCEPTIONS as exc:
last_error = exc
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
if attempt < max_attempts:
time.sleep(delay_seconds * (2 ** (attempt - 1)))
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
if jitter_seconds > 0:
sleep_for += random.uniform(0, jitter_seconds)
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
time.sleep(sleep_for)
if last_error is not None:
raise last_error
raise RuntimeError("Retry wrapper failed without a captured exception")

88
iaai_scraper/scraper.py
View File

@@ -1,5 +1,6 @@
import logging
import time
import uuid
from pathlib import Path
from typing import Any
@@ -9,7 +10,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from .browser import BrowserFactory, HumanPacer, NetworkCapture
from .core.config import Settings, settings
from .core.logs import setup_logging
from .core.logs import set_trace_id, setup_logging
from .core.retry import retryable
from .core.utils import save_to_json
from .parsing.mapper import CarMapper
@@ -24,8 +25,11 @@ logger = logging.getLogger("iaai_scraper.scraper")
class IAAIScraper:
def __init__(self, runtime_settings: Settings | None = None) -> None:
# Базовые зависимости и сервисы скрапера.
self.settings = runtime_settings or settings
setup_logging(self.settings.log_level, self.settings.log_file)
self.trace_id = uuid.uuid4().hex[:12]
set_trace_id(self.trace_id)
self.playwright = None
self.browser = None
self.context: BrowserContext | None = None
@@ -38,20 +42,51 @@ class IAAIScraper:
# browser lifecycle
def _new_trace_id(self, prefix: str) -> str:
# Новый trace id для отдельной операции.
trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
self.trace_id = trace_id
set_trace_id(trace_id)
return trace_id
def __enter__(self) -> "IAAIScraper":
self.playwright = sync_playwright().start()
self.browser = self.browser_factory.create_browser(self.playwright)
if self.playwright is None:
self.playwright = sync_playwright().start()
if self.browser is None:
self.browser = self.browser_factory.create_browser(self.playwright)
return self
def __exit__(self, exc_type, exc, tb) -> None:
if self.context:
self.context.close()
if self.browser:
self.browser.close()
if self.playwright:
self.playwright.stop()
self.close()
def close(self) -> None:
# Закрываем ресурсы аккуратно, даже если Playwright уже в ошибке.
if self.context is not None:
try:
self.context.close()
except PlaywrightError:
pass
finally:
self.context = None
if self.browser is not None:
try:
self.browser.close()
except PlaywrightError:
pass
finally:
self.browser = None
if self.playwright is not None:
try:
self.playwright.stop()
except PlaywrightError:
pass
finally:
self.playwright = None
def _new_context(self, storage_state: str | None = None) -> BrowserContext:
# Контекст пересоздаётся, чтобы не тянуть старое состояние страницы.
if self.browser is None:
self.__enter__()
if self.context:
self.context.close()
self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state)
@@ -68,6 +103,7 @@ class IAAIScraper:
# listing
def collect_listing(self, make: str | None = None, model: str | None = None):
# Собираем ссылки карточек с публичного листинга.
page = self._get_unauthenticated_page()
try:
@@ -87,8 +123,11 @@ class IAAIScraper:
# scrape
@retryable(max_attempts=3)
@retryable(max_attempts=3, jitter_seconds=0.25)
def open_vehicle_page(self, vehicle_url: str):
# Лёгкое открытие страницы без сетевого дампа и сохранения в БД.
trace_id = self._new_trace_id("open")
started_at = time.perf_counter()
page = self._get_page()
try:
self.pacer.before_vehicle_open()
@@ -107,15 +146,17 @@ class IAAIScraper:
dom_text = ""
parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []})
return {
"trace_id": trace_id,
"source_url": vehicle_url,
"opened_in_gentle_mode": True,
"fetched_at_epoch": int(time.time()),
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
**parsed,
}
finally:
page.close()
@retryable(max_attempts=3)
@retryable(max_attempts=3, jitter_seconds=0.25)
def scrape_vehicle_detail(self, vehicle_url: str):
"""Открыть страницу, перехватить JSON, вернуть данные."""
page = self._get_page()
@@ -125,6 +166,9 @@ class IAAIScraper:
page.close()
def _scrape_on_page(self, page: Page, vehicle_url: str):
# Полный проход по карточке с network capture и маппингом в DB-модель.
trace_id = self._new_trace_id("scrape")
started_at = time.perf_counter()
capture = NetworkCapture(self.settings)
capture.attach(page)
@@ -150,8 +194,10 @@ class IAAIScraper:
)
result = {
"trace_id": trace_id,
"source_url": vehicle_url,
"fetched_at_epoch": int(time.time()),
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"network": network_dump,
**parsed,
"db_record": db_record.model_dump(mode="json"),
@@ -165,6 +211,9 @@ class IAAIScraper:
def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
"""Scrape + upsert одного авто."""
# Один URL -> один scrape -> один upsert.
trace_id = self._new_trace_id("sync-vehicle")
started_at = time.perf_counter()
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane=lane)
ids_fetched = 1
@@ -184,11 +233,13 @@ class IAAIScraper:
images_upserted = int(upsert.get("images_upserted", 0))
status = "success"
return {
"trace_id": trace_id,
"status": status,
"run_id": run_id,
"vehicle_url": vehicle_url,
"db_action": upsert.get("action"),
"images_upserted": images_upserted,
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"db_record": db_record,
}
except Exception as exc:
@@ -207,11 +258,16 @@ class IAAIScraper:
error_summary=error_summary,
)
def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars"):
def sync_listing(self, make: str | None = None, model: str | None = None, lane: str = "iaai_cars", limit: int | None = None):
"""Листинг + sync всех найденных машин."""
# Массовая синхронизация с общим run_id и сбором ошибок.
trace_id = self._new_trace_id("sync-listing")
started_at = time.perf_counter()
self.persistence.create_tables()
listing = self.collect_listing(make=make, model=model)
vehicle_urls = list(listing.get("vehicle_urls", []))
if limit is not None:
vehicle_urls = vehicle_urls[:max(0, limit)]
run_id = self.persistence.start_sync_run(lane=lane)
cars_upserted = 0
cars_failed = 0
@@ -225,6 +281,7 @@ class IAAIScraper:
try:
for index, vehicle_url in enumerate(vehicle_urls, start=1):
try:
# Повторно используем страницу, чтобы не создавать лишний overhead.
logger.info("[%d/%d] Scraping %s", index, total, vehicle_url)
scrape_result = self._scrape_on_page(page, vehicle_url)
db_record = scrape_result.get("db_record")
@@ -242,6 +299,7 @@ class IAAIScraper:
record.year or "?", record.price or "N/A", img_count,
)
except Exception as exc:
# После сбоя пересоздаём page, чтобы не остаться в битом состоянии.
cars_failed += 1
failures.append({"vehicle_url": vehicle_url, "error": str(exc)})
logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc)
@@ -252,7 +310,7 @@ class IAAIScraper:
page = self._get_page()
if index < total:
time.sleep(1.0)
self.pacer.between_vehicles()
finally:
try:
page.close()
@@ -275,18 +333,21 @@ class IAAIScraper:
run_id, cars_upserted, total, cars_failed, images_upserted,
)
return {
"trace_id": trace_id,
"status": status,
"run_id": run_id,
"listing": listing,
"cars_upserted": cars_upserted,
"cars_failed": cars_failed,
"images_upserted": images_upserted,
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"failures": failures,
}
# scheduler
def run_scheduled(self) -> None:
# Простой бесконечный цикл без внешнего планировщика.
interval = self.settings.scheduler_interval_minutes * 60
logger.info(
"Scheduler started: syncing every %d minutes",
@@ -332,6 +393,7 @@ class IAAIScraper:
def _warm_page(self, page: Page) -> None:
"""Scroll для lazy-load."""
# Небольшой прогрев страницы для ленивых блоков и картинок.
rounds = max(0, self.settings.gentle.warm_scroll_rounds)
pause_seconds = max(0.1, self.settings.gentle.scroll_pause_ms / 1000)
for _ in range(rounds):

10
iaai_scraper/storage/listing.py
View File

@@ -15,6 +15,7 @@ VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
@dataclass(slots=True)
class ListingVehicleLink:
# Ссылка на одну карточку из листинга.
href: str
title: str = ""
lot_number: str | None = None
@@ -22,6 +23,7 @@ class ListingVehicleLink:
@dataclass(slots=True)
class ListingPageResult:
# Результат сбора ссылок с одной страницы листинга.
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
@@ -31,10 +33,12 @@ class ListingPageResult:
class ListingCollector:
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
# Сборщик ссылок из публичного листинга IAAI.
self.settings = settings
self.pacer = pacer
def open_cars_listing(self, page: Page) -> None:
# Открываем листинг и ждём появления ссылок на карточки.
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
try:
@@ -48,6 +52,7 @@ class ListingCollector:
self.pacer.after_listing_open()
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
# Пытаемся применить make/model фильтры через UI.
applied = {"make": None, "model": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
@@ -58,6 +63,7 @@ class ListingCollector:
return applied
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Собираем ссылки только с текущей страницы.
anchors = page.locator("a[href*='/VehicleDetail/']")
total = min(anchors.count(), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
@@ -80,6 +86,7 @@ class ListingCollector:
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def go_to_next_page(self, page: Page) -> bool:
# Переход на следующую страницу, если кнопка доступна.
selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
for selector in selectors:
locator = page.locator(selector).first
@@ -101,6 +108,7 @@ class ListingCollector:
return False
def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, object]:
# Последовательно собираем ссылки с учётом лимитов и пагинации.
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
pages: list[dict[str, object]] = []
@@ -121,6 +129,7 @@ class ListingCollector:
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
# Пробуем несколько селекторов для одного и того же фильтра.
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
@@ -138,6 +147,7 @@ class ListingCollector:
@staticmethod
def _has_next_page(page: Page) -> bool:
# Грубая проверка наличия кнопки next.
for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
if page.locator(selector).count() > 0:
return True