304 lines
10 KiB
Python
304 lines
10 KiB
Python
import logging
|
|
import re
|
|
import time
|
|
from dataclasses import asdict, dataclass, field
|
|
from typing import Any
|
|
from urllib.parse import urljoin
|
|
|
|
from playwright.sync_api import Page
|
|
|
|
from .pace import HumanPacer
|
|
from ..core.config import Settings
|
|
from ..core.utils import first_non_empty
|
|
|
|
logger = logging.getLogger("iaai_scraper.listing")
|
|
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingVehicleLink:
|
|
href: str
|
|
title: str = ""
|
|
lot_number: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingPageResult:
|
|
source_url: str
|
|
page_number: int
|
|
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
|
|
pagination_available: bool = False
|
|
next_page_detected: bool = False
|
|
|
|
|
|
class ListingCollector:
|
|
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
|
|
self.settings = settings
|
|
self.pacer = pacer
|
|
|
|
def open_cars_listing(self, page: Page) -> None:
|
|
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
|
|
url = self.settings.listing.cars_url
|
|
last_err = None
|
|
for attempt in range(3):
|
|
try:
|
|
page.goto(url, wait_until="commit", timeout=60_000)
|
|
last_err = None
|
|
break
|
|
except Exception as e:
|
|
last_err = e
|
|
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
|
|
# Небольшой backoff при ошибках открытия листинга.
|
|
time.sleep(5 * (attempt + 1))
|
|
if last_err:
|
|
logger.warning("All goto attempts failed, trying JS navigation")
|
|
try:
|
|
page.evaluate(f"window.location.href = '{url}'")
|
|
except Exception:
|
|
pass
|
|
# Быстрая проверка готовности страницы.
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=12_000)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000)
|
|
except Exception:
|
|
# Короткая пауза вместо длинного sleep.
|
|
time.sleep(0.25)
|
|
logger.info("Listing page URL: %s", page.url)
|
|
self.pacer.after_listing_open()
|
|
|
|
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
|
|
applied = {"make": None, "model": None}
|
|
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
|
|
applied["make"] = make
|
|
self.pacer.after_filter_action()
|
|
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
|
|
applied["model"] = model
|
|
self.pacer.after_filter_action()
|
|
return applied
|
|
|
|
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
|
|
# Считываем ссылки одним проходом по DOM.
|
|
try:
|
|
raw_items = page.eval_on_selector_all(
|
|
"a[href*='/VehicleDetail/']",
|
|
"""
|
|
(nodes) => nodes.map((a) => ({
|
|
href: a.getAttribute('href') || '',
|
|
title: a.getAttribute('title') || '',
|
|
text: (a.textContent || '').trim(),
|
|
}))
|
|
""",
|
|
)
|
|
except Exception as exc:
|
|
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
|
|
raw_items = []
|
|
total = min(len(raw_items), self.settings.listing.page_link_limit)
|
|
links: list[ListingVehicleLink] = []
|
|
seen: set[str] = set()
|
|
for idx in range(total):
|
|
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
|
|
href = str(item.get("href") or "")
|
|
match = VEHICLE_HREF_RE.search(href)
|
|
if not match:
|
|
continue
|
|
lot_number = match.group(1)
|
|
absolute = urljoin(self.settings.home_url, match.group(0))
|
|
if absolute in seen:
|
|
continue
|
|
seen.add(absolute)
|
|
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
|
|
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
|
|
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
|
break
|
|
next_page_detected = self._has_next_page(page)
|
|
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
|
|
|
|
def go_to_next_page(self, page: Page) -> bool:
|
|
# Запоминаем первую ссылку текущей страницы для определения смены контента.
|
|
old_first_href = ""
|
|
try:
|
|
first_link = page.locator("a[href*='/VehicleDetail/']").first
|
|
if first_link.count() > 0:
|
|
old_first_href = first_link.get_attribute("href") or ""
|
|
except Exception:
|
|
pass
|
|
|
|
selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
|
|
for selector in selectors:
|
|
locator = page.locator(selector).first
|
|
if locator.count() == 0:
|
|
continue
|
|
try:
|
|
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
|
|
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
|
|
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
|
|
except Exception:
|
|
continue
|
|
if disabled or aria_disabled == "true" or "disabled" in classes:
|
|
continue
|
|
try:
|
|
self.pacer.move_mouse_to(page, locator)
|
|
locator.click(timeout=8000)
|
|
except Exception:
|
|
continue
|
|
|
|
# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
|
|
if old_first_href:
|
|
try:
|
|
page.wait_for_function(
|
|
f"""() => {{
|
|
const a = document.querySelector("a[href*='/VehicleDetail/']");
|
|
return a && a.getAttribute('href') !== '{old_first_href}';
|
|
}}""",
|
|
timeout=8000,
|
|
)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
|
|
except Exception:
|
|
pass
|
|
self.pacer.after_page_change()
|
|
return True
|
|
return False
|
|
|
|
def collect_listing_links(
|
|
self,
|
|
page: Page,
|
|
*,
|
|
make: str | None = None,
|
|
model: str | None = None,
|
|
known_origin_ids: set[str] | None = None,
|
|
max_duration_seconds: float | None = None,
|
|
) -> dict[str, Any]:
|
|
self.open_cars_listing(page)
|
|
applied_filters = self.apply_filters(page, make=make, model=model)
|
|
started_at = time.perf_counter()
|
|
truncated_by_time_budget = False
|
|
pages: list[dict[str, object]] = []
|
|
all_links: list[str] = []
|
|
early_stopped = False
|
|
threshold = self.settings.listing.early_stop_threshold
|
|
|
|
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
|
|
if max_duration_seconds is not None and max_duration_seconds > 0:
|
|
elapsed = time.perf_counter() - started_at
|
|
if elapsed >= max_duration_seconds:
|
|
truncated_by_time_budget = True
|
|
logger.warning(
|
|
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
|
|
page_number,
|
|
elapsed,
|
|
max_duration_seconds,
|
|
)
|
|
break
|
|
page_result = self.collect_current_page(page, page_number=page_number)
|
|
pages.append({
|
|
"page_number": page_result.page_number,
|
|
"source_url": page_result.source_url,
|
|
"links_found": len(page_result.vehicle_links),
|
|
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
|
|
"next_page_detected": page_result.next_page_detected,
|
|
})
|
|
for item in page_result.vehicle_links:
|
|
if item.href not in all_links:
|
|
all_links.append(item.href)
|
|
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
|
|
break
|
|
|
|
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
|
|
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
|
|
if (
|
|
known_origin_ids is not None
|
|
and threshold > 0.0
|
|
and page_result.vehicle_links
|
|
):
|
|
page_known = sum(
|
|
1 for item in page_result.vehicle_links
|
|
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
|
|
)
|
|
page_new = len(page_result.vehicle_links) - page_known
|
|
ratio = page_known / len(page_result.vehicle_links)
|
|
# Останавливаемся только если нет новых И порог превышен
|
|
if ratio >= threshold and page_new == 0:
|
|
logger.info(
|
|
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
|
|
page_number, ratio * 100, page_known,
|
|
len(page_result.vehicle_links), threshold * 100,
|
|
)
|
|
early_stopped = True
|
|
break
|
|
elif page_new > 0 and ratio >= threshold:
|
|
logger.info(
|
|
"Page %d: %.0f%% known but %d new found — продолжаем",
|
|
page_number, ratio * 100, page_new,
|
|
)
|
|
|
|
if (
|
|
len(all_links) >= self.settings.listing.max_vehicles_per_run
|
|
or self.settings.listing.collect_current_page_only
|
|
or not self.settings.listing.include_pagination
|
|
or not page_result.next_page_detected
|
|
):
|
|
break
|
|
if not self.go_to_next_page(page):
|
|
break
|
|
|
|
return {
|
|
"listing_url": self.settings.listing.cars_url,
|
|
"applied_filters": applied_filters,
|
|
"pages_collected": len(pages),
|
|
"vehicles_collected": len(all_links),
|
|
"vehicle_urls": all_links,
|
|
"early_stopped": early_stopped,
|
|
"truncated_by_time_budget": truncated_by_time_budget,
|
|
"pages": pages,
|
|
"strategy": {
|
|
"sequential": True,
|
|
"collect_current_page_only": self.settings.listing.collect_current_page_only,
|
|
"include_pagination": self.settings.listing.include_pagination,
|
|
"max_pages_per_run": self.settings.listing.max_pages_per_run,
|
|
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
|
|
"early_stop_threshold": threshold,
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
|
|
for selector in selectors:
|
|
locator = page.locator(selector).first
|
|
if locator.count() == 0:
|
|
continue
|
|
try:
|
|
locator.click()
|
|
locator.fill(value)
|
|
page.keyboard.press("Enter")
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
|
|
except Exception:
|
|
pass
|
|
return True
|
|
except Exception:
|
|
continue
|
|
return False
|
|
|
|
@staticmethod
|
|
def _has_next_page(page: Page) -> bool:
|
|
for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
|
|
if page.locator(selector).count() > 0:
|
|
return True
|
|
return False
|