827 lines
29 KiB
Python
827 lines
29 KiB
Python
import logging
|
||
import re
|
||
import time
|
||
from dataclasses import asdict, dataclass, field
|
||
from typing import Any
|
||
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
|
||
|
||
from playwright.sync_api import Page
|
||
|
||
from .pace import HumanPacer
|
||
from ..core.config import Settings
|
||
|
||
logger = logging.getLogger("iaai_scraper.listing")
|
||
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
|
||
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
|
||
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
|
||
"button:has-text('Accept All')",
|
||
"button:has-text('Accept all')",
|
||
"button:has-text('I Agree')",
|
||
"button:has-text('Agree')",
|
||
"button:has-text('Only necessary')",
|
||
"button:has-text('Только необходимые')",
|
||
"button:has-text('Принять все')",
|
||
"[id*='accept']",
|
||
"[class*='accept']",
|
||
)
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class ListingVehicleLink:
|
||
href: str
|
||
title: str = ""
|
||
lot_number: str | None = None
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class ListingPageResult:
|
||
source_url: str
|
||
page_number: int
|
||
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
|
||
pagination_available: bool = False
|
||
next_page_detected: bool = False
|
||
|
||
|
||
class ListingCollector:
|
||
_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
|
||
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
|
||
"a[aria-label*='Next']",
|
||
"button[aria-label*='Next']",
|
||
"a[aria-label*='next']",
|
||
"button[aria-label*='next']",
|
||
"a[title*='Next']",
|
||
"button[title*='Next']",
|
||
"a[title*='next']",
|
||
"button[title*='next']",
|
||
"a[rel='next']",
|
||
"link[rel='next']",
|
||
"a.pagination-next",
|
||
"button.pagination-next",
|
||
"a.next",
|
||
"button.next",
|
||
"a:has-text('Next')",
|
||
"button:has-text('Next')",
|
||
"a:has-text('NEXT')",
|
||
"button:has-text('NEXT')",
|
||
"a:has-text('›')",
|
||
"button:has-text('›')",
|
||
"a:has-text('»')",
|
||
"button:has-text('»')",
|
||
"a:has(img[src*='icon-arrow-right'])",
|
||
"button:has(img[src*='icon-arrow-right'])",
|
||
"a:has(img[src*='arrow-right'])",
|
||
"button:has(img[src*='arrow-right'])",
|
||
)
|
||
|
||
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
|
||
self.settings = settings
|
||
self.pacer = pacer
|
||
|
||
@staticmethod
|
||
def _get_current_page_number(page: Page) -> int | None:
|
||
try:
|
||
value = page.evaluate(
|
||
"""
|
||
() => {
|
||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
|
||
const current = controls.find((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||
});
|
||
if (current) {
|
||
return parseInt((current.textContent || '').trim(), 10);
|
||
}
|
||
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
|
||
return match ? parseInt(match[1], 10) : null;
|
||
}
|
||
"""
|
||
)
|
||
return int(value) if value is not None else None
|
||
except Exception:
|
||
return None
|
||
|
||
@staticmethod
|
||
def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
|
||
try:
|
||
values = page.evaluate(
|
||
"""
|
||
(limit) => {
|
||
return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
|
||
.map((el) => (el.getAttribute('href') || '').trim())
|
||
.filter(Boolean)
|
||
.slice(0, limit);
|
||
}
|
||
""",
|
||
limit,
|
||
)
|
||
if not isinstance(values, list):
|
||
return tuple()
|
||
return tuple(str(value) for value in values if value)
|
||
except Exception:
|
||
return tuple()
|
||
|
||
@staticmethod
|
||
def _wait_for_navigation_result(
|
||
page: Page,
|
||
old_first_href: str,
|
||
expected_page_number: int | None,
|
||
*,
|
||
old_fingerprint: tuple[str, ...] = (),
|
||
) -> bool:
|
||
if old_first_href:
|
||
try:
|
||
page.wait_for_function(
|
||
f"""() => {{
|
||
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
|
||
return a && a.getAttribute('href') !== '{old_first_href}';
|
||
}}""",
|
||
timeout=12000,
|
||
)
|
||
return True
|
||
except Exception:
|
||
pass
|
||
|
||
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
|
||
if expected_page_number is not None:
|
||
current_page = ListingCollector._get_current_page_number(page)
|
||
if current_page == expected_page_number and (
|
||
not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
|
||
):
|
||
return True
|
||
|
||
try:
|
||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||
except Exception:
|
||
pass
|
||
|
||
if not new_fingerprint:
|
||
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
|
||
current_page = ListingCollector._get_current_page_number(page)
|
||
if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
|
||
return current_page is None or expected_page_number is None or current_page == expected_page_number
|
||
if expected_page_number is not None and current_page == expected_page_number:
|
||
return not old_fingerprint
|
||
|
||
return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
|
||
|
||
def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
|
||
try:
|
||
href = page.evaluate(
|
||
"""
|
||
(expectedPageNumber) => {
|
||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||
const disabled = (el) => {
|
||
if (!el) return true;
|
||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||
};
|
||
|
||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
|
||
.filter((el) => visible(el) && !disabled(el));
|
||
|
||
const cleanHref = (el) => {
|
||
const href = (el?.getAttribute('href') || '').trim();
|
||
if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
|
||
return '';
|
||
}
|
||
return href;
|
||
};
|
||
|
||
if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
|
||
const numeric = controls.find((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
|
||
});
|
||
const numericHref = cleanHref(numeric);
|
||
if (numericHref) {
|
||
return numericHref;
|
||
}
|
||
}
|
||
|
||
const explicitNext = controls.find((el) => {
|
||
const text = (el.textContent || '').trim().toLowerCase();
|
||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
|
||
});
|
||
return cleanHref(explicitNext);
|
||
}
|
||
""",
|
||
expected_page_number,
|
||
)
|
||
if not href:
|
||
return None
|
||
return urljoin(page.url or self.settings.home_url, str(href))
|
||
except Exception:
|
||
return None
|
||
|
||
@staticmethod
|
||
def _build_listing_page_url(url: str, page_number: int) -> str:
|
||
parsed = urlparse(url)
|
||
query_items = [
|
||
(key, value)
|
||
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
|
||
if key.lower() not in {"page", "pagenumber", "currentpage"}
|
||
]
|
||
query_items.append(("page", str(page_number)))
|
||
return urlunparse(parsed._replace(query=urlencode(query_items)))
|
||
|
||
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
|
||
url = url_override or self.settings.listing.cars_url
|
||
logger.info("Opening cars listing page: %s", url)
|
||
last_err = None
|
||
for attempt in range(3):
|
||
try:
|
||
page.goto(url, wait_until="commit", timeout=60_000)
|
||
last_err = None
|
||
break
|
||
except Exception as e:
|
||
last_err = e
|
||
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
|
||
# Небольшой backoff при ошибках открытия листинга.
|
||
time.sleep(5 * (attempt + 1))
|
||
if last_err:
|
||
logger.warning("All goto attempts failed, trying JS navigation")
|
||
try:
|
||
page.evaluate(f"window.location.href = '{url}'")
|
||
except Exception:
|
||
pass
|
||
# Быстрая проверка готовности страницы.
|
||
try:
|
||
page.wait_for_load_state("domcontentloaded", timeout=12_000)
|
||
except Exception:
|
||
pass
|
||
self._accept_cookie_banner(page)
|
||
self._wait_for_listing_content(page)
|
||
logger.info("Listing page URL: %s", page.url)
|
||
self.pacer.after_listing_open()
|
||
|
||
def _accept_cookie_banner(self, page: Page) -> None:
|
||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||
locator = page.locator(selector).first
|
||
try:
|
||
if locator.count() == 0:
|
||
continue
|
||
if not locator.is_visible(timeout=500):
|
||
continue
|
||
locator.click(timeout=2_000)
|
||
logger.info("Accepted cookie banner using selector: %s", selector)
|
||
try:
|
||
page.wait_for_load_state("domcontentloaded", timeout=3_000)
|
||
except Exception:
|
||
pass
|
||
return
|
||
except Exception:
|
||
continue
|
||
|
||
def _wait_for_listing_content(self, page: Page) -> None:
|
||
try:
|
||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
|
||
return
|
||
except Exception:
|
||
pass
|
||
|
||
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
|
||
try:
|
||
page.wait_for_function(
|
||
"""() => {
|
||
const html = document.documentElement?.innerHTML || '';
|
||
const text = document.body?.innerText || '';
|
||
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
|
||
}""",
|
||
timeout=12_000,
|
||
)
|
||
except Exception:
|
||
# Короткая пауза вместо длинного sleep.
|
||
time.sleep(1.0)
|
||
|
||
def _get_listing_html(self, page: Page, page_number: int) -> str:
|
||
try:
|
||
return page.locator("html").inner_html(timeout=5_000)
|
||
except Exception as exc:
|
||
logger.warning("listing html read failed on page %d: %s", page_number, exc)
|
||
return ""
|
||
|
||
def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
|
||
if not html:
|
||
return False
|
||
normalized = html.replace("\\/", "/")
|
||
if re.search(
|
||
r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
|
||
normalized,
|
||
re.IGNORECASE,
|
||
):
|
||
return True
|
||
if current_page_number is not None:
|
||
next_page = current_page_number + 1
|
||
if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
|
||
return True
|
||
if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
|
||
return True
|
||
return False
|
||
|
||
def apply_filters(
|
||
self,
|
||
page: Page,
|
||
make: str | None = None,
|
||
model: str | None = None,
|
||
year_min: int | None = None,
|
||
year_max: int | None = None,
|
||
) -> dict[str, str | int | None]:
|
||
applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
|
||
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
|
||
applied["make"] = make
|
||
self.pacer.after_filter_action()
|
||
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
|
||
applied["model"] = model
|
||
self.pacer.after_filter_action()
|
||
if year_min is not None or year_max is not None:
|
||
if self._apply_year_range(page, year_min, year_max):
|
||
applied["year_min"] = year_min
|
||
applied["year_max"] = year_max
|
||
self.pacer.after_filter_action()
|
||
return applied
|
||
|
||
def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
|
||
"""Заполняет поля фильтра Year и нажимает Apply Year."""
|
||
if year_min is None and year_max is None:
|
||
return False
|
||
try:
|
||
success = page.evaluate(
|
||
"""([yearMin, yearMax]) => {
|
||
const inputs = Array.from(document.querySelectorAll('input'));
|
||
const yearInputs = inputs.filter(inp => {
|
||
const v = parseInt(inp.value, 10);
|
||
return !isNaN(v) && v >= 1900 && v <= 2100;
|
||
});
|
||
if (yearInputs.length < 2) return false;
|
||
yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
|
||
const setVal = (el, val) => {
|
||
const setter = Object.getOwnPropertyDescriptor(
|
||
HTMLInputElement.prototype, 'value'
|
||
).set;
|
||
setter.call(el, String(val));
|
||
el.dispatchEvent(new Event('input', {bubbles: true}));
|
||
el.dispatchEvent(new Event('change', {bubbles: true}));
|
||
};
|
||
if (yearMin !== null) setVal(yearInputs[0], yearMin);
|
||
if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
|
||
const container = yearInputs[0].closest(
|
||
'[class*="filter"], [class*="year"], section, fieldset'
|
||
) || yearInputs[0].parentElement.parentElement;
|
||
if (container) {
|
||
const btn = Array.from(container.querySelectorAll(
|
||
'button, a, [role="button"], span[class*="apply"]'
|
||
)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
|
||
if (btn) { btn.click(); return true; }
|
||
}
|
||
yearInputs[yearInputs.length - 1].dispatchEvent(
|
||
new KeyboardEvent('keydown', {
|
||
key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
|
||
})
|
||
);
|
||
return true;
|
||
}""",
|
||
[year_min, year_max],
|
||
)
|
||
if success:
|
||
try:
|
||
page.wait_for_load_state("domcontentloaded", timeout=15_000)
|
||
except Exception:
|
||
pass
|
||
self._wait_for_listing_content(page)
|
||
logger.info("Applied year range filter: %s — %s", year_min, year_max)
|
||
return True
|
||
except Exception as exc:
|
||
logger.warning("Failed to apply year range filter: %s", exc)
|
||
return False
|
||
|
||
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
|
||
# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
|
||
self._accept_cookie_banner(page)
|
||
self._wait_for_listing_content(page)
|
||
links: list[ListingVehicleLink] = []
|
||
seen: set[str] = set()
|
||
html = self._get_listing_html(page, page_number)
|
||
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
|
||
if absolute in seen:
|
||
continue
|
||
seen.add(absolute)
|
||
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
|
||
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
||
break
|
||
|
||
if links:
|
||
logger.info(
|
||
"Page %d: recovered %d vehicle links from HTML fallback",
|
||
page_number,
|
||
len(links),
|
||
)
|
||
|
||
next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
|
||
if not next_page_detected and not html:
|
||
next_page_detected = self._has_next_page(page)
|
||
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
|
||
|
||
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
|
||
if not html:
|
||
return []
|
||
|
||
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
|
||
normalized = html.replace("\\/", "/")
|
||
found: list[tuple[str, str]] = []
|
||
seen: set[str] = set()
|
||
|
||
for match in VEHICLE_HREF_RE.finditer(normalized):
|
||
lot_number = match.group(1)
|
||
absolute = urljoin(self.settings.home_url, match.group(0))
|
||
if absolute in seen:
|
||
continue
|
||
seen.add(absolute)
|
||
found.append((absolute, lot_number))
|
||
if len(found) >= self.settings.listing.page_link_limit:
|
||
break
|
||
|
||
return found
|
||
|
||
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
|
||
# Запоминаем первую ссылку текущей страницы для определения смены контента.
|
||
old_first_href = ""
|
||
old_fingerprint: tuple[str, ...] = tuple()
|
||
try:
|
||
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
|
||
if first_link.count() > 0:
|
||
old_first_href = first_link.get_attribute("href") or ""
|
||
except Exception:
|
||
pass
|
||
old_fingerprint = self._get_vehicle_link_fingerprint(page)
|
||
|
||
if expected_page_number is not None and expected_page_number > 1:
|
||
direct_page_url = self._build_listing_page_url(
|
||
page.url or self.settings.listing.cars_url,
|
||
expected_page_number,
|
||
)
|
||
try:
|
||
logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
|
||
page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
|
||
if self._wait_for_navigation_result(
|
||
page,
|
||
old_first_href,
|
||
expected_page_number,
|
||
old_fingerprint=old_fingerprint,
|
||
):
|
||
self.pacer.after_page_change()
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
|
||
|
||
next_href = self._extract_next_page_href(page, expected_page_number)
|
||
if next_href:
|
||
try:
|
||
logger.debug("Navigating directly to next listing page: %s", next_href)
|
||
page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
|
||
if self._wait_for_navigation_result(
|
||
page,
|
||
old_first_href,
|
||
expected_page_number,
|
||
old_fingerprint=old_fingerprint,
|
||
):
|
||
self.pacer.after_page_change()
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
|
||
|
||
if (
|
||
expected_page_number is not None
|
||
and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
|
||
):
|
||
logger.warning(
|
||
"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
|
||
expected_page_number,
|
||
)
|
||
return False
|
||
|
||
for selector in self._NEXT_PAGE_SELECTORS:
|
||
locator = page.locator(selector).first
|
||
if locator.count() == 0:
|
||
continue
|
||
try:
|
||
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
|
||
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
|
||
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
|
||
except Exception:
|
||
continue
|
||
if disabled or aria_disabled == "true" or "disabled" in classes:
|
||
continue
|
||
try:
|
||
self.pacer.move_mouse_to(page, locator)
|
||
locator.click(timeout=8000)
|
||
except Exception:
|
||
continue
|
||
|
||
if self._wait_for_navigation_result(
|
||
page,
|
||
old_first_href,
|
||
expected_page_number,
|
||
old_fingerprint=old_fingerprint,
|
||
):
|
||
self.pacer.after_page_change()
|
||
return True
|
||
|
||
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
|
||
# + стрелка с иконкой, без явного текста Next.
|
||
try:
|
||
clicked = bool(page.evaluate(
|
||
"""
|
||
() => {
|
||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||
const disabled = (el) => {
|
||
if (!el) return true;
|
||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||
};
|
||
|
||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
|
||
.filter((el) => visible(el) && !disabled(el));
|
||
|
||
const current = controls.find((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||
});
|
||
|
||
if (current) {
|
||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||
const nextNumber = controls.find((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||
});
|
||
if (nextNumber) {
|
||
nextNumber.click();
|
||
return true;
|
||
}
|
||
}
|
||
|
||
const iconNext = controls.find((el) => {
|
||
const text = (el.textContent || '').trim().toLowerCase();
|
||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||
});
|
||
|
||
if (iconNext) {
|
||
iconNext.click();
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
"""
|
||
))
|
||
if clicked:
|
||
if self._wait_for_navigation_result(
|
||
page,
|
||
old_first_href,
|
||
expected_page_number,
|
||
old_fingerprint=old_fingerprint,
|
||
):
|
||
self.pacer.after_page_change()
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
|
||
|
||
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
|
||
try:
|
||
clicked = bool(page.evaluate(
|
||
"""
|
||
() => {
|
||
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
|
||
for (const el of candidates) {
|
||
const text = (el.textContent || '').trim().toLowerCase();
|
||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
|
||
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||
if (!disabled && visible && looksNext) {
|
||
el.click();
|
||
return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
"""
|
||
))
|
||
if clicked:
|
||
if self._wait_for_navigation_result(
|
||
page,
|
||
old_first_href,
|
||
expected_page_number,
|
||
old_fingerprint=old_fingerprint,
|
||
):
|
||
self.pacer.after_page_change()
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug("JS next-page fallback failed: %s", exc)
|
||
|
||
logger.warning("Could not navigate to next page from %s", page.url)
|
||
return False
|
||
|
||
def collect_listing_links(
|
||
self,
|
||
page: Page,
|
||
*,
|
||
make: str | None = None,
|
||
model: str | None = None,
|
||
known_origin_ids: set[str] | None = None,
|
||
max_duration_seconds: float | None = None,
|
||
) -> dict[str, Any]:
|
||
self.open_cars_listing(page)
|
||
applied_filters = self.apply_filters(page, make=make, model=model)
|
||
started_at = time.perf_counter()
|
||
truncated_by_time_budget = False
|
||
pages: list[dict[str, object]] = []
|
||
all_links: list[str] = []
|
||
early_stopped = False
|
||
threshold = self.settings.listing.early_stop_threshold
|
||
|
||
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
|
||
if max_duration_seconds is not None and max_duration_seconds > 0:
|
||
elapsed = time.perf_counter() - started_at
|
||
if elapsed >= max_duration_seconds:
|
||
truncated_by_time_budget = True
|
||
logger.warning(
|
||
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
|
||
page_number,
|
||
elapsed,
|
||
max_duration_seconds,
|
||
)
|
||
break
|
||
page_result = self.collect_current_page(page, page_number=page_number)
|
||
pages.append({
|
||
"page_number": page_result.page_number,
|
||
"source_url": page_result.source_url,
|
||
"links_found": len(page_result.vehicle_links),
|
||
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
|
||
"next_page_detected": page_result.next_page_detected,
|
||
})
|
||
for item in page_result.vehicle_links:
|
||
if item.href not in all_links:
|
||
all_links.append(item.href)
|
||
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
|
||
break
|
||
|
||
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
|
||
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
|
||
if (
|
||
known_origin_ids is not None
|
||
and threshold > 0.0
|
||
and page_result.vehicle_links
|
||
):
|
||
page_known = sum(
|
||
1 for item in page_result.vehicle_links
|
||
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
|
||
)
|
||
page_new = len(page_result.vehicle_links) - page_known
|
||
ratio = page_known / len(page_result.vehicle_links)
|
||
# Останавливаемся только если нет новых И порог превышен
|
||
if ratio >= threshold and page_new == 0:
|
||
logger.info(
|
||
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
|
||
page_number, ratio * 100, page_known,
|
||
len(page_result.vehicle_links), threshold * 100,
|
||
)
|
||
early_stopped = True
|
||
break
|
||
elif page_new > 0 and ratio >= threshold:
|
||
logger.info(
|
||
"Page %d: %.0f%% known but %d new found — продолжаем",
|
||
page_number, ratio * 100, page_new,
|
||
)
|
||
|
||
if (
|
||
len(all_links) >= self.settings.listing.max_vehicles_per_run
|
||
or self.settings.listing.collect_current_page_only
|
||
or not self.settings.listing.include_pagination
|
||
):
|
||
break
|
||
|
||
blind_page_probe = not page_result.next_page_detected
|
||
if not self.go_to_next_page(page, expected_page_number=page_number + 1):
|
||
if blind_page_probe:
|
||
logger.info(
|
||
"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
|
||
page_number,
|
||
page_number + 1,
|
||
)
|
||
break
|
||
break
|
||
|
||
return {
|
||
"listing_url": self.settings.listing.cars_url,
|
||
"applied_filters": applied_filters,
|
||
"pages_collected": len(pages),
|
||
"vehicles_collected": len(all_links),
|
||
"vehicle_urls": all_links,
|
||
"early_stopped": early_stopped,
|
||
"truncated_by_time_budget": truncated_by_time_budget,
|
||
"pages": pages,
|
||
"strategy": {
|
||
"sequential": True,
|
||
"collect_current_page_only": self.settings.listing.collect_current_page_only,
|
||
"include_pagination": self.settings.listing.include_pagination,
|
||
"max_pages_per_run": self.settings.listing.max_pages_per_run,
|
||
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
|
||
"early_stop_threshold": threshold,
|
||
},
|
||
}
|
||
|
||
@staticmethod
|
||
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
|
||
for selector in selectors:
|
||
locator = page.locator(selector).first
|
||
if locator.count() == 0:
|
||
continue
|
||
try:
|
||
locator.click()
|
||
locator.fill(value)
|
||
page.keyboard.press("Enter")
|
||
try:
|
||
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||
except Exception:
|
||
pass
|
||
return True
|
||
except Exception:
|
||
continue
|
||
return False
|
||
|
||
@staticmethod
|
||
def _has_next_page(page: Page) -> bool:
|
||
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
|
||
if page.locator(selector).count() > 0:
|
||
return True
|
||
try:
|
||
return bool(page.evaluate(
|
||
"""
|
||
() => {
|
||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
|
||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||
return !disabled && visible(el);
|
||
});
|
||
|
||
const hasExplicitNext = controls.some((el) => {
|
||
const text = (el.textContent || '').trim().toLowerCase();
|
||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
|
||
});
|
||
|
||
if (hasExplicitNext) {
|
||
return true;
|
||
}
|
||
|
||
const current = controls.find((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||
});
|
||
|
||
if (!current) {
|
||
return false;
|
||
}
|
||
|
||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||
return controls.some((el) => {
|
||
const text = (el.textContent || '').trim();
|
||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||
});
|
||
}
|
||
"""
|
||
))
|
||
except Exception:
|
||
return False
|
||
return False
|