qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix scraper flow

Browse Source
This commit is contained in:
qananasikq
2026-04-22 21:40:40 +03:00
parent 133c125e9c
commit d5d6ba8b7c
33 changed files with 2632 additions and 2811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

322
iaai_scraper/browser/listing.py
View File

@@ -3,12 +3,13 @@ import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
@@ -43,7 +44,6 @@ class ListingPageResult:
class ListingCollector:
_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
"a[aria-label*='Next']",
"button[aria-label*='Next']",
@@ -103,33 +103,7 @@ class ListingCollector:
return None
@staticmethod
def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
try:
values = page.evaluate(
"""
(limit) => {
return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
.map((el) => (el.getAttribute('href') || '').trim())
.filter(Boolean)
.slice(0, limit);
}
""",
limit,
)
if not isinstance(values, list):
return tuple()
return tuple(str(value) for value in values if value)
except Exception:
return tuple()
@staticmethod
def _wait_for_navigation_result(
page: Page,
old_first_href: str,
expected_page_number: int | None,
*,
old_fingerprint: tuple[str, ...] = (),
) -> bool:
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
if old_first_href:
try:
page.wait_for_function(
@@ -143,12 +117,9 @@ class ListingCollector:
except Exception:
pass
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number and (
not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
):
if current_page == expected_page_number:
return True
try:
@@ -156,81 +127,33 @@ class ListingCollector:
except Exception:
pass
if not new_fingerprint:
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
current_page = ListingCollector._get_current_page_number(page)
if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
return current_page is None or expected_page_number is None or current_page == expected_page_number
if expected_page_number is not None and current_page == expected_page_number:
return not old_fingerprint
return True
return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
try:
href = page.evaluate(
"""
(expectedPageNumber) => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
};
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
.filter((el) => visible(el) && !disabled(el));
const cleanHref = (el) => {
const href = (el?.getAttribute('href') || '').trim();
if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
return '';
}
return href;
};
if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
const numeric = controls.find((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
});
const numericHref = cleanHref(numeric);
if (numericHref) {
return numericHref;
}
}
const explicitNext = controls.find((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
});
return cleanHref(explicitNext);
}
""",
expected_page_number,
)
if not href:
return None
return urljoin(page.url or self.settings.home_url, str(href))
except Exception:
return None
return not old_first_href
@staticmethod
def _build_listing_page_url(url: str, page_number: int) -> str:
parsed = urlparse(url)
query_items = [
(key, value)
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
if key.lower() not in {"page", "pagenumber", "currentpage"}
]
query_items.append(("page", str(page_number)))
return urlunparse(parsed._replace(query=urlencode(query_items)))
def has_page_number(page: Page, target_page_number: int) -> bool:
try:
return bool(page.evaluate(
"""
(targetPageNumber) => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
return controls.some((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
});
}
""",
target_page_number,
))
except Exception:
return False
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
url = url_override or self.settings.listing.cars_url
@@ -301,31 +224,6 @@ class ListingCollector:
# Короткая пауза вместо длинного sleep.
time.sleep(1.0)
def _get_listing_html(self, page: Page, page_number: int) -> str:
try:
return page.locator("html").inner_html(timeout=5_000)
except Exception as exc:
logger.warning("listing html read failed on page %d: %s", page_number, exc)
return ""
def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
if not html:
return False
normalized = html.replace("\\/", "/")
if re.search(
r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
normalized,
re.IGNORECASE,
):
return True
if current_page_number is not None:
next_page = current_page_number + 1
if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
return True
if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
return True
return False
def apply_filters(
self,
page: Page,
@@ -403,30 +301,74 @@ class ListingCollector:
return False
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
# Считываем ссылки одним проходом по DOM.
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
try:
raw_items = page.eval_on_selector_all(
"a[href], [data-href], [href]",
"""
(nodes) => nodes.map((node) => ({
href:
node.getAttribute('href') ||
node.getAttribute('data-href') ||
node.getAttribute('data-url') ||
'',
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
text: (node.textContent || '').trim(),
}))
""",
)
except Exception as exc:
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
raw_items = []
total = min(len(raw_items), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
html = self._get_listing_html(page, page_number)
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
for idx in range(total):
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
href = str(item.get("href") or "")
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
page.wait_for_timeout(1_500)
except Exception:
pass
try:
html = page.content()
except Exception as exc:
logger.debug("page.content() failed on page %d: %s", page_number, exc)
html = ""
next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
if not next_page_detected and not html:
next_page_detected = self._has_next_page(page)
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
@@ -453,59 +395,12 @@ class ListingCollector:
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
old_fingerprint: tuple[str, ...] = tuple()
try:
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
if first_link.count() > 0:
old_first_href = first_link.get_attribute("href") or ""
except Exception:
pass
old_fingerprint = self._get_vehicle_link_fingerprint(page)
if expected_page_number is not None and expected_page_number > 1:
direct_page_url = self._build_listing_page_url(
page.url or self.settings.listing.cars_url,
expected_page_number,
)
try:
logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
next_href = self._extract_next_page_href(page, expected_page_number)
if next_href:
try:
logger.debug("Navigating directly to next listing page: %s", next_href)
page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
if (
expected_page_number is not None
and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
):
logger.warning(
"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
expected_page_number,
)
return False
for selector in self._NEXT_PAGE_SELECTORS:
locator = page.locator(selector).first
@@ -525,12 +420,7 @@ class ListingCollector:
except Exception:
continue
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
@@ -590,12 +480,7 @@ class ListingCollector:
"""
))
if clicked:
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
@@ -626,12 +511,7 @@ class ListingCollector:
"""
))
if clicked:
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
@@ -716,18 +596,10 @@ class ListingCollector:
len(all_links) >= self.settings.listing.max_vehicles_per_run
or self.settings.listing.collect_current_page_only
or not self.settings.listing.include_pagination
or not page_result.next_page_detected
):
break
blind_page_probe = not page_result.next_page_detected
if not self.go_to_next_page(page, expected_page_number=page_number + 1):
if blind_page_probe:
logger.info(
"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
page_number,
page_number + 1,
)
break
if not self.go_to_next_page(page):
break
return {
@@ -775,8 +647,22 @@ class ListingCollector:
@staticmethod
def _has_next_page(page: Page) -> bool:
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
if page.locator(selector).count() > 0:
return True
locator = page.locator(selector)
count = locator.count()
if count == 0:
continue
# Проверяем, что хотя бы один элемент не disabled.
# Disabled "Next" на последней странице не означает наличия следующей.
for i in range(min(count, 3)):
try:
el = locator.nth(i)
disabled_attr = el.get_attribute("disabled", timeout=300)
aria_disabled = el.get_attribute("aria-disabled", timeout=300)
cls = (el.get_attribute("class", timeout=300) or "").lower()
if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
return True
except Exception:
continue
try:
return bool(page.evaluate(
"""