import logging import re import time from dataclasses import asdict, dataclass, field from typing import Any from urllib.parse import urljoin from playwright.sync_api import Page from .pace import HumanPacer from ..core.config import Settings from ..core.utils import first_non_empty logger = logging.getLogger("iaai_scraper.listing") VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) @dataclass(slots=True) class ListingVehicleLink: href: str title: str = "" lot_number: str | None = None @dataclass(slots=True) class ListingPageResult: source_url: str page_number: int vehicle_links: list[ListingVehicleLink] = field(default_factory=list) pagination_available: bool = False next_page_detected: bool = False class ListingCollector: def __init__(self, settings: Settings, pacer: HumanPacer) -> None: self.settings = settings self.pacer = pacer def open_cars_listing(self, page: Page) -> None: logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) url = self.settings.listing.cars_url last_err = None for attempt in range(3): try: page.goto(url, wait_until="commit", timeout=60_000) last_err = None break except Exception as e: last_err = e logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) # Небольшой backoff при ошибках открытия листинга. time.sleep(5 * (attempt + 1)) if last_err: logger.warning("All goto attempts failed, trying JS navigation") try: page.evaluate(f"window.location.href = '{url}'") except Exception: pass # Быстрая проверка готовности страницы. try: page.wait_for_load_state("domcontentloaded", timeout=12_000) except Exception: pass try: page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000) except Exception: # Короткая пауза вместо длинного sleep. time.sleep(0.25) logger.info("Listing page URL: %s", page.url) self.pacer.after_listing_open() def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: applied = {"make": None, "model": None} if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): applied["make"] = make self.pacer.after_filter_action() if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): applied["model"] = model self.pacer.after_filter_action() return applied def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: # Считываем ссылки одним проходом по DOM. raw_items = page.eval_on_selector_all( "a[href*='/VehicleDetail/']", """ (nodes) => nodes.map((a) => ({ href: a.getAttribute('href') || '', title: a.getAttribute('title') || '', text: (a.textContent || '').trim(), })) """, ) total = min(len(raw_items), self.settings.listing.page_link_limit) links: list[ListingVehicleLink] = [] seen: set[str] = set() for idx in range(total): item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} href = str(item.get("href") or "") match = VEHICLE_HREF_RE.search(href) if not match: continue lot_number = match.group(1) absolute = urljoin(self.settings.home_url, match.group(0)) if absolute in seen: continue seen.add(absolute) title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) if len(links) >= self.settings.listing.max_vehicles_per_run: break next_page_detected = self._has_next_page(page) return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) def go_to_next_page(self, page: Page) -> bool: # Запоминаем первую ссылку текущей страницы для определения смены контента. old_first_href = "" try: first_link = page.locator("a[href*='/VehicleDetail/']").first if first_link.count() > 0: old_first_href = first_link.get_attribute("href") or "" except Exception: pass selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] for selector in selectors: locator = page.locator(selector).first if locator.count() == 0: continue disabled = (locator.get_attribute("disabled") or "").lower() aria_disabled = (locator.get_attribute("aria-disabled") or "").lower() classes = (locator.get_attribute("class") or "").lower() if disabled or aria_disabled == "true" or "disabled" in classes: continue self.pacer.move_mouse_to(page, locator) locator.click() # Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться. if old_first_href: try: page.wait_for_function( f"""() => {{ const a = document.querySelector("a[href*='/VehicleDetail/']"); return a && a.getAttribute('href') !== '{old_first_href}'; }}""", timeout=8000, ) except Exception: pass else: try: page.wait_for_load_state("domcontentloaded", timeout=15000) except Exception: pass try: page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) except Exception: pass self.pacer.after_page_change() return True return False def collect_listing_links( self, page: Page, *, make: str | None = None, model: str | None = None, known_origin_ids: set[str] | None = None, ) -> dict[str, Any]: self.open_cars_listing(page) applied_filters = self.apply_filters(page, make=make, model=model) pages: list[dict[str, object]] = [] all_links: list[str] = [] early_stopped = False threshold = self.settings.listing.early_stop_threshold for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): page_result = self.collect_current_page(page, page_number=page_number) pages.append({ "page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected, }) for item in page_result.vehicle_links: if item.href not in all_links: all_links.append(item.href) if len(all_links) >= self.settings.listing.max_vehicles_per_run: break # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). if ( known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links ): page_known = sum( 1 for item in page_result.vehicle_links if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids ) page_new = len(page_result.vehicle_links) - page_known ratio = page_known / len(page_result.vehicle_links) # Останавливаемся только если нет новых И порог превышен if ratio >= threshold and page_new == 0: logger.info( "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", page_number, ratio * 100, page_known, len(page_result.vehicle_links), threshold * 100, ) early_stopped = True break elif page_new > 0 and ratio >= threshold: logger.info( "Page %d: %.0f%% known but %d new found — продолжаем", page_number, ratio * 100, page_new, ) if ( len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected ): break if not self.go_to_next_page(page): break return { "listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "early_stopped": early_stopped, "pages": pages, "strategy": { "sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, "early_stop_threshold": threshold, }, } @staticmethod def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: for selector in selectors: locator = page.locator(selector).first if locator.count() == 0: continue try: locator.click() locator.fill(value) page.keyboard.press("Enter") try: page.wait_for_load_state("domcontentloaded", timeout=15000) except Exception: pass try: page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) except Exception: pass return True except Exception: continue return False @staticmethod def _has_next_page(page: Page) -> bool: for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: if page.locator(selector).count() > 0: return True return False