qananasikq/Openlane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix parsing db and tests

Browse Source
This commit is contained in:
qananasikq
2026-04-08 13:31:02 +03:00
parent f96e5ca5f8
commit 0010abdf08
9 changed files with 188 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
iaai_scraper/parsing/parser.py
View File

@@ -74,6 +74,7 @@ class VehicleParser:
}
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
# Вытаскиваем пары label -> value из плоского текста страницы.
result: dict[str, str] = {}
if not dom_text:
return result
@@ -93,6 +94,7 @@ class VehicleParser:
return result
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
# Title используется как запасной источник year/make/model.
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
if title_match:
@@ -108,6 +110,7 @@ class VehicleParser:
return result
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
# Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
page_html = page_html or ""
dom_text = dom_text or ""
network_dump = network_dump or {}
@@ -122,6 +125,7 @@ class VehicleParser:
summary: dict[str, Any] = {"source_url": vehicle_url}
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
# Для каждого поля собираем кандидатов из всех доступных источников.
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, candidate_keys))
@@ -156,6 +160,7 @@ class VehicleParser:
summary["current_bid"] = prices[1]
embedded = self._extract_embedded_json(page_html)
# Embedded JSON добирает поля, которых не было в DOM и XHR.
for item in embedded:
p = item.get("payload")
if isinstance(p, (dict, list)):
@@ -175,6 +180,7 @@ class VehicleParser:
}
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
# Группируем сырой результат по смысловым блокам для маппера.
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
return {
"vehicle_core": {
@@ -217,6 +223,7 @@ class VehicleParser:
}
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
# Раскладываем observed endpoints по категориям.
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
for item in responses:
url = item.get("url", "")
@@ -235,12 +242,14 @@ class VehicleParser:
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
# Короткие признаки того, что страница была доступна нормально.
endpoints = [item.get("url", "") for item in responses]
return {
"vin_visible": bool(summary.get("vin")),
"images_visible": bool(summary.get("image_urls")),
"network_json_count": len(responses),
"possible_captcha": False,
"possible_antibot": False,
"observed_endpoints": endpoints[:20],
}
@@ -280,6 +289,7 @@ class VehicleParser:
@staticmethod
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
# Ищем inline JSON в script-тегах.
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
extracted: list[dict[str, Any]] = []
for script_text in scripts:
@@ -294,6 +304,7 @@ class VehicleParser:
@staticmethod
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
# Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
vehicle_key = ""
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
if key_match:
@@ -302,13 +313,15 @@ class VehicleParser:
for payload in payloads:
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
flat: list[str] = []
seen_flat: set[str] = set()
for item in found:
if isinstance(item, str) and item.startswith("http"):
cleaned = html_module.unescape(item)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in flat:
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
elif isinstance(item, list):
for child in item:
@@ -317,26 +330,31 @@ class VehicleParser:
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in flat:
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
for match in re.finditer(pattern, html or "", re.IGNORECASE):
url = html_module.unescape(match.group(1).strip())
if not url.startswith("http") or url in flat:
if not url.startswith("http") or url in seen_flat:
continue
lowered = url.lower()
if vehicle_key and vehicle_key in url:
seen_flat.add(url)
flat.append(url)
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
if vehicle_key and vehicle_key not in url:
continue
seen_flat.add(url)
flat.append(url)
if vehicle_key:
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
cleaned = html_module.unescape(url)
if cleaned not in flat and vehicle_key in cleaned:
if cleaned not in seen_flat and vehicle_key in cleaned:
seen_flat.add(cleaned)
flat.append(cleaned)
filtered: list[str] = []
# Отбрасываем служебные и заведомо нецелевые ссылки.
for url in flat:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -348,6 +366,7 @@ class VehicleParser:
@staticmethod
def _dom_hints(text: str) -> dict[str, Any]:
# Быстрые текстовые признаки полезных данных или anti-bot страницы.
lowered = (text or "").lower()
return {
"has_buy_now_text": "buy now" in lowered,
@@ -355,4 +374,6 @@ class VehicleParser:
"has_damage_text": "damage" in lowered,
"has_vin_text": "vin" in lowered,
"has_title_text": "title" in lowered,
"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
}