a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
279 lines
9.7 KiB
Python
279 lines
9.7 KiB
Python
import logging
|
|
import re
|
|
import time
|
|
from dataclasses import asdict, dataclass, field
|
|
from typing import Any
|
|
from urllib.parse import urljoin
|
|
|
|
from playwright.sync_api import Page
|
|
|
|
from .pace import HumanPacer
|
|
from ..core.config import Settings
|
|
from ..core.utils import first_non_empty
|
|
|
|
logger = logging.getLogger("iaai_scraper.listing")
|
|
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingVehicleLink:
|
|
href: str
|
|
title: str = ""
|
|
lot_number: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ListingPageResult:
|
|
source_url: str
|
|
page_number: int
|
|
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
|
|
pagination_available: bool = False
|
|
next_page_detected: bool = False
|
|
|
|
|
|
class ListingCollector:
|
|
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
|
|
self.settings = settings
|
|
self.pacer = pacer
|
|
|
|
def open_cars_listing(self, page: Page) -> None:
|
|
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
|
|
url = self.settings.listing.cars_url
|
|
last_err = None
|
|
for attempt in range(3):
|
|
try:
|
|
page.goto(url, wait_until="commit", timeout=60_000)
|
|
last_err = None
|
|
break
|
|
except Exception as e:
|
|
last_err = e
|
|
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
|
|
# Небольшой backoff при ошибках открытия листинга.
|
|
time.sleep(5 * (attempt + 1))
|
|
if last_err:
|
|
logger.warning("All goto attempts failed, trying JS navigation")
|
|
try:
|
|
page.evaluate(f"window.location.href = '{url}'")
|
|
except Exception:
|
|
pass
|
|
# Быстрая проверка готовности страницы.
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=12_000)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000)
|
|
except Exception:
|
|
# Короткая пауза вместо длинного sleep.
|
|
time.sleep(0.25)
|
|
logger.info("Listing page URL: %s", page.url)
|
|
self.pacer.after_listing_open()
|
|
|
|
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
|
|
applied = {"make": None, "model": None}
|
|
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
|
|
applied["make"] = make
|
|
self.pacer.after_filter_action()
|
|
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
|
|
applied["model"] = model
|
|
self.pacer.after_filter_action()
|
|
return applied
|
|
|
|
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
|
|
# Считываем ссылки одним проходом по DOM.
|
|
raw_items = page.eval_on_selector_all(
|
|
"a[href*='/VehicleDetail/']",
|
|
"""
|
|
(nodes) => nodes.map((a) => ({
|
|
href: a.getAttribute('href') || '',
|
|
title: a.getAttribute('title') || '',
|
|
text: (a.textContent || '').trim(),
|
|
}))
|
|
""",
|
|
)
|
|
total = min(len(raw_items), self.settings.listing.page_link_limit)
|
|
links: list[ListingVehicleLink] = []
|
|
seen: set[str] = set()
|
|
for idx in range(total):
|
|
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
|
|
href = str(item.get("href") or "")
|
|
match = VEHICLE_HREF_RE.search(href)
|
|
if not match:
|
|
continue
|
|
lot_number = match.group(1)
|
|
absolute = urljoin(self.settings.home_url, match.group(0))
|
|
if absolute in seen:
|
|
continue
|
|
seen.add(absolute)
|
|
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
|
|
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
|
|
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
|
break
|
|
next_page_detected = self._has_next_page(page)
|
|
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
|
|
|
|
def go_to_next_page(self, page: Page) -> bool:
|
|
# Запоминаем первую ссылку текущей страницы для определения смены контента.
|
|
old_first_href = ""
|
|
try:
|
|
first_link = page.locator("a[href*='/VehicleDetail/']").first
|
|
if first_link.count() > 0:
|
|
old_first_href = first_link.get_attribute("href") or ""
|
|
except Exception:
|
|
pass
|
|
|
|
selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
|
|
for selector in selectors:
|
|
locator = page.locator(selector).first
|
|
if locator.count() == 0:
|
|
continue
|
|
disabled = (locator.get_attribute("disabled") or "").lower()
|
|
aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
|
|
classes = (locator.get_attribute("class") or "").lower()
|
|
if disabled or aria_disabled == "true" or "disabled" in classes:
|
|
continue
|
|
self.pacer.move_mouse_to(page, locator)
|
|
locator.click()
|
|
|
|
# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
|
|
if old_first_href:
|
|
try:
|
|
page.wait_for_function(
|
|
f"""() => {{
|
|
const a = document.querySelector("a[href*='/VehicleDetail/']");
|
|
return a && a.getAttribute('href') !== '{old_first_href}';
|
|
}}""",
|
|
timeout=8000,
|
|
)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
|
|
except Exception:
|
|
pass
|
|
self.pacer.after_page_change()
|
|
return True
|
|
return False
|
|
|
|
def collect_listing_links(
|
|
self,
|
|
page: Page,
|
|
*,
|
|
make: str | None = None,
|
|
model: str | None = None,
|
|
known_origin_ids: set[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
self.open_cars_listing(page)
|
|
applied_filters = self.apply_filters(page, make=make, model=model)
|
|
pages: list[dict[str, object]] = []
|
|
all_links: list[str] = []
|
|
early_stopped = False
|
|
threshold = self.settings.listing.early_stop_threshold
|
|
|
|
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
|
|
page_result = self.collect_current_page(page, page_number=page_number)
|
|
pages.append({
|
|
"page_number": page_result.page_number,
|
|
"source_url": page_result.source_url,
|
|
"links_found": len(page_result.vehicle_links),
|
|
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
|
|
"next_page_detected": page_result.next_page_detected,
|
|
})
|
|
for item in page_result.vehicle_links:
|
|
if item.href not in all_links:
|
|
all_links.append(item.href)
|
|
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
|
|
break
|
|
|
|
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
|
|
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
|
|
if (
|
|
known_origin_ids is not None
|
|
and threshold > 0.0
|
|
and page_result.vehicle_links
|
|
):
|
|
page_known = sum(
|
|
1 for item in page_result.vehicle_links
|
|
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
|
|
)
|
|
page_new = len(page_result.vehicle_links) - page_known
|
|
ratio = page_known / len(page_result.vehicle_links)
|
|
# Останавливаемся только если нет новых И порог превышен
|
|
if ratio >= threshold and page_new == 0:
|
|
logger.info(
|
|
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
|
|
page_number, ratio * 100, page_known,
|
|
len(page_result.vehicle_links), threshold * 100,
|
|
)
|
|
early_stopped = True
|
|
break
|
|
elif page_new > 0 and ratio >= threshold:
|
|
logger.info(
|
|
"Page %d: %.0f%% known but %d new found — продолжаем",
|
|
page_number, ratio * 100, page_new,
|
|
)
|
|
|
|
if (
|
|
len(all_links) >= self.settings.listing.max_vehicles_per_run
|
|
or self.settings.listing.collect_current_page_only
|
|
or not self.settings.listing.include_pagination
|
|
or not page_result.next_page_detected
|
|
):
|
|
break
|
|
if not self.go_to_next_page(page):
|
|
break
|
|
|
|
return {
|
|
"listing_url": self.settings.listing.cars_url,
|
|
"applied_filters": applied_filters,
|
|
"pages_collected": len(pages),
|
|
"vehicles_collected": len(all_links),
|
|
"vehicle_urls": all_links,
|
|
"early_stopped": early_stopped,
|
|
"pages": pages,
|
|
"strategy": {
|
|
"sequential": True,
|
|
"collect_current_page_only": self.settings.listing.collect_current_page_only,
|
|
"include_pagination": self.settings.listing.include_pagination,
|
|
"max_pages_per_run": self.settings.listing.max_pages_per_run,
|
|
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
|
|
"early_stop_threshold": threshold,
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
|
|
for selector in selectors:
|
|
locator = page.locator(selector).first
|
|
if locator.count() == 0:
|
|
continue
|
|
try:
|
|
locator.click()
|
|
locator.fill(value)
|
|
page.keyboard.press("Enter")
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
|
|
except Exception:
|
|
pass
|
|
return True
|
|
except Exception:
|
|
continue
|
|
return False
|
|
|
|
@staticmethod
|
|
def _has_next_page(page: Page) -> bool:
|
|
for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
|
|
if page.locator(selector).count() > 0:
|
|
return True
|
|
return False
|