qananasikq/Openlane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup and fix runtime bugs

Browse Source
This commit is contained in:
qananasikq
2026-04-08 22:54:16 +03:00
parent 90bd4756b3
commit 6e97991c68
21 changed files with 220 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
iaai_scraper/parsing/parser.py
View File

@@ -74,7 +74,6 @@ class VehicleParser:
}
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
# Вытаскиваем пары label -> value из плоского текста страницы.
result: dict[str, str] = {}
if not dom_text:
return result
@@ -94,7 +93,6 @@ class VehicleParser:
return result
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
# Title используется как запасной источник year/make/model.
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
if title_match:
@@ -110,7 +108,6 @@ class VehicleParser:
return result
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
# Собираем итоговую структуру из DOM, title, embedded JSON и network payloads.
page_html = page_html or ""
dom_text = dom_text or ""
network_dump = network_dump or {}
@@ -125,7 +122,6 @@ class VehicleParser:
summary: dict[str, Any] = {"source_url": vehicle_url}
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
# Для каждого поля собираем кандидатов из всех доступных источников.
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, candidate_keys))
@@ -160,7 +156,6 @@ class VehicleParser:
summary["current_bid"] = prices[1]
embedded = self._extract_embedded_json(page_html)
# Embedded JSON добирает поля, которых не было в DOM и XHR.
for item in embedded:
p = item.get("payload")
if isinstance(p, (dict, list)):
@@ -180,7 +175,6 @@ class VehicleParser:
}
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
# Группируем сырой результат по смысловым блокам для маппера.
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
return {
"vehicle_core": {
@@ -223,7 +217,6 @@ class VehicleParser:
}
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
# Раскладываем observed endpoints по категориям.
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
for item in responses:
url = item.get("url", "")
@@ -242,7 +235,6 @@ class VehicleParser:
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
# Короткие признаки того, что страница была доступна нормально.
endpoints = [item.get("url", "") for item in responses]
return {
"vin_visible": bool(summary.get("vin")),
@@ -289,7 +281,6 @@ class VehicleParser:
@staticmethod
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
# Ищем inline JSON в script-тегах.
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
extracted: list[dict[str, Any]] = []
for script_text in scripts:
@@ -304,7 +295,6 @@ class VehicleParser:
@staticmethod
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
# Собираем и дедуплицируем ссылки на изображения из JSON и HTML.
vehicle_key = ""
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
if key_match:
@@ -354,7 +344,6 @@ class VehicleParser:
seen_flat.add(cleaned)
flat.append(cleaned)
filtered: list[str] = []
# Отбрасываем служебные и заведомо нецелевые ссылки.
for url in flat:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
@@ -366,7 +355,6 @@ class VehicleParser:
@staticmethod
def _dom_hints(text: str) -> dict[str, Any]:
# Быстрые текстовые признаки полезных данных или anti-bot страницы.
lowered = (text or "").lower()
return {
"has_buy_now_text": "buy now" in lowered,