import logging
import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
"button:has-text('Accept All')",
"button:has-text('Accept all')",
"button:has-text('I Agree')",
"button:has-text('Agree')",
"button:has-text('Only necessary')",
"button:has-text('Только необходимые')",
"button:has-text('Принять все')",
"[id*='accept']",
"[class*='accept']",
)
@dataclass(slots=True)
class ListingVehicleLink:
href: str
title: str = ""
lot_number: str | None = None
@dataclass(slots=True)
class ListingPageResult:
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
pagination_available: bool = False
next_page_detected: bool = False
class ListingCollector:
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
"a[aria-label*='Next']",
"button[aria-label*='Next']",
"a[aria-label*='next']",
"button[aria-label*='next']",
"a[title*='Next']",
"button[title*='Next']",
"a[title*='next']",
"button[title*='next']",
"a[rel='next']",
"link[rel='next']",
"a.pagination-next",
"button.pagination-next",
"a.next",
"button.next",
"a:has-text('Next')",
"button:has-text('Next')",
"a:has-text('NEXT')",
"button:has-text('NEXT')",
"a:has-text('›')",
"button:has-text('›')",
"a:has-text('»')",
"button:has-text('»')",
"a:has(img[src*='icon-arrow-right'])",
"button:has(img[src*='icon-arrow-right'])",
"a:has(img[src*='arrow-right'])",
"button:has(img[src*='arrow-right'])",
)
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
self.settings = settings
self.pacer = pacer
def open_cars_listing(self, page: Page) -> None:
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
url = self.settings.listing.cars_url
last_err = None
for attempt in range(3):
try:
page.goto(url, wait_until="commit", timeout=60_000)
last_err = None
break
except Exception as e:
last_err = e
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
# Небольшой backoff при ошибках открытия листинга.
time.sleep(5 * (attempt + 1))
if last_err:
logger.warning("All goto attempts failed, trying JS navigation")
try:
page.evaluate(f"window.location.href = '{url}'")
except Exception:
pass
# Быстрая проверка готовности страницы.
try:
page.wait_for_load_state("domcontentloaded", timeout=12_000)
except Exception:
pass
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
logger.info("Listing page URL: %s", page.url)
self.pacer.after_listing_open()
def _accept_cookie_banner(self, page: Page) -> None:
for selector in COOKIE_ACCEPT_SELECTORS:
locator = page.locator(selector).first
try:
if locator.count() == 0:
continue
if not locator.is_visible(timeout=500):
continue
locator.click(timeout=2_000)
logger.info("Accepted cookie banner using selector: %s", selector)
try:
page.wait_for_load_state("domcontentloaded", timeout=3_000)
except Exception:
pass
return
except Exception:
continue
def _wait_for_listing_content(self, page: Page) -> None:
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
return
except Exception:
pass
# Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM.
try:
page.wait_for_function(
"""() => {
const html = document.documentElement?.innerHTML || '';
const text = document.body?.innerText || '';
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
}""",
timeout=12_000,
)
except Exception:
# Короткая пауза вместо длинного sleep.
time.sleep(1.0)
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
applied = {"make": None, "model": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
self.pacer.after_filter_action()
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
applied["model"] = model
self.pacer.after_filter_action()
return applied
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Считываем ссылки одним проходом по DOM.
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
try:
raw_items = page.eval_on_selector_all(
"a[href], [data-href], [href]",
"""
(nodes) => nodes.map((node) => ({
href:
node.getAttribute('href') ||
node.getAttribute('data-href') ||
node.getAttribute('data-url') ||
'',
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
text: (node.textContent || '').trim(),
}))
""",
)
except Exception as exc:
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
raw_items = []
total = min(len(raw_items), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
href = str(item.get("href") or "")
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
# Fallback: на IAAI ссылки иногда не рендерятся как ,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
page.wait_for_timeout(1_500)
except Exception:
pass
try:
html = page.content()
except Exception as exc:
logger.debug("page.content() failed on page %d: %s", page_number, exc)
html = ""
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
if not html:
return []
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
normalized = html.replace("\\/", "/")
found: list[tuple[str, str]] = []
seen: set[str] = set()
for match in VEHICLE_HREF_RE.finditer(normalized):
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
found.append((absolute, lot_number))
if len(found) >= self.settings.listing.page_link_limit:
break
return found
def go_to_next_page(self, page: Page) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
try:
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
if first_link.count() > 0:
old_first_href = first_link.get_attribute("href") or ""
except Exception:
pass
for selector in self._NEXT_PAGE_SELECTORS:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
except Exception:
continue
if disabled or aria_disabled == "true" or "disabled" in classes:
continue
try:
self.pacer.move_mouse_to(page, locator)
locator.click(timeout=8000)
except Exception:
continue
# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
self.pacer.after_page_change()
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
try:
clicked = bool(page.evaluate(
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
};
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
.filter((el) => visible(el) && !disabled(el));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
const currentPage = parseInt((current.textContent || '').trim(), 10);
const nextNumber = controls.find((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
if (nextNumber) {
nextNumber.click();
return true;
}
}
const iconNext = controls.find((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
});
if (iconNext) {
iconNext.click();
return true;
}
return false;
}
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=12000)
except Exception:
pass
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
try:
clicked = bool(page.evaluate(
"""
() => {
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
for (const el of candidates) {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
if (!disabled && visible && looksNext) {
el.click();
return true;
}
}
return false;
}
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("JS next-page fallback failed: %s", exc)
logger.warning("Could not navigate to next page from %s", page.url)
return False
def collect_listing_links(
self,
page: Page,
*,
make: str | None = None,
model: str | None = None,
known_origin_ids: set[str] | None = None,
max_duration_seconds: float | None = None,
) -> dict[str, Any]:
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
started_at = time.perf_counter()
truncated_by_time_budget = False
pages: list[dict[str, object]] = []
all_links: list[str] = []
early_stopped = False
threshold = self.settings.listing.early_stop_threshold
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
if max_duration_seconds is not None and max_duration_seconds > 0:
elapsed = time.perf_counter() - started_at
if elapsed >= max_duration_seconds:
truncated_by_time_budget = True
logger.warning(
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
page_number,
elapsed,
max_duration_seconds,
)
break
page_result = self.collect_current_page(page, page_number=page_number)
pages.append({
"page_number": page_result.page_number,
"source_url": page_result.source_url,
"links_found": len(page_result.vehicle_links),
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
"next_page_detected": page_result.next_page_detected,
})
for item in page_result.vehicle_links:
if item.href not in all_links:
all_links.append(item.href)
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
break
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
if (
known_origin_ids is not None
and threshold > 0.0
and page_result.vehicle_links
):
page_known = sum(
1 for item in page_result.vehicle_links
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
)
page_new = len(page_result.vehicle_links) - page_known
ratio = page_known / len(page_result.vehicle_links)
# Останавливаемся только если нет новых И порог превышен
if ratio >= threshold and page_new == 0:
logger.info(
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
page_number, ratio * 100, page_known,
len(page_result.vehicle_links), threshold * 100,
)
early_stopped = True
break
elif page_new > 0 and ratio >= threshold:
logger.info(
"Page %d: %.0f%% known but %d new found — продолжаем",
page_number, ratio * 100, page_new,
)
if (
len(all_links) >= self.settings.listing.max_vehicles_per_run
or self.settings.listing.collect_current_page_only
or not self.settings.listing.include_pagination
or not page_result.next_page_detected
):
break
if not self.go_to_next_page(page):
break
return {
"listing_url": self.settings.listing.cars_url,
"applied_filters": applied_filters,
"pages_collected": len(pages),
"vehicles_collected": len(all_links),
"vehicle_urls": all_links,
"early_stopped": early_stopped,
"truncated_by_time_budget": truncated_by_time_budget,
"pages": pages,
"strategy": {
"sequential": True,
"collect_current_page_only": self.settings.listing.collect_current_page_only,
"include_pagination": self.settings.listing.include_pagination,
"max_pages_per_run": self.settings.listing.max_pages_per_run,
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
"early_stop_threshold": threshold,
},
}
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
locator.click()
locator.fill(value)
page.keyboard.press("Enter")
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
return True
except Exception:
continue
return False
@staticmethod
def _has_next_page(page: Page) -> bool:
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
if page.locator(selector).count() > 0:
return True
try:
return bool(page.evaluate(
"""
() => Array.from(document.querySelectorAll('a,button,[role="button"]')).some((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
const looksNumericNext = /^\d+$/.test(text);
return !disabled && visible && (rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || looksNumericNext || ['next', '›', '»', '>'].includes(text));
})
"""
))
except Exception:
return False
return False