qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add checkpoint logic and tests

Browse Source
This commit is contained in:
qananasikq
2026-04-15 11:10:32 +03:00
parent d9111be2d2
commit acbb045b6e
8 changed files with 432 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
iaai_scraper/browser/listing.py
View File

@@ -77,6 +77,62 @@ class ListingCollector:
self.settings = settings
self.pacer = pacer
@staticmethod
def _get_current_page_number(page: Page) -> int | None:
try:
value = page.evaluate(
"""
() => {
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
return parseInt((current.textContent || '').trim(), 10);
}
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
return match ? parseInt(match[1], 10) : null;
}
"""
)
return int(value) if value is not None else None
except Exception:
return None
@staticmethod
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
return True
except Exception:
pass
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number:
return True
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
current_page = ListingCollector._get_current_page_number(page)
if expected_page_number is not None and current_page == expected_page_number:
return True
return not old_first_href
def open_cars_listing(self, page: Page) -> None:
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
url = self.settings.listing.cars_url
@@ -248,7 +304,7 @@ class ListingCollector:
return found
def go_to_next_page(self, page: Page) -> bool:
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
try:
@@ -276,30 +332,9 @@ class ListingCollector:
except Exception:
continue
# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
@@ -357,24 +392,9 @@ class ListingCollector:
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=12000)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
@@ -403,19 +423,9 @@ class ListingCollector:
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("JS next-page fallback failed: %s", exc)