Enrich search cards

mobilede_scraper/mobile_de/flight.py

@@ -2,9 +2,104 @@ from __future__ import annotations
 
 import json
 import re
+from html import unescape
+from html.parser import HTMLParser
 from typing import Any
+from urllib.parse import parse_qs, urlsplit
 
 NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+SEARCH_CARD_TEST_ID_RE = re.compile(r"^base-result-listing-\d+$")
+VOID_HTML_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"}
+
+
+class _SearchCardEnrichmentParser(HTMLParser):
+    """Collect title/subtitle from SSR cards while preserving the listing-ID relation."""
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.enrichment_by_id: dict[str, dict[str, str]] = {}
+        self._card: dict[str, Any] | None = None
+        self._card_depth = 0
+        self._title_container_depth: int | None = None
+        self._title_text_depth = 0
+
+    @staticmethod
+    def _listing_id_from_href(href: str | None) -> str | None:
+        if not href:
+            return None
+        values = parse_qs(urlsplit(unescape(href)).query).get("id", [])
+        return str(values[0]).strip() if values and str(values[0]).strip() else None
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attributes = dict(attrs)
+        test_id = attributes.get("data-testid") or ""
+        is_void = tag.lower() in VOID_HTML_TAGS
+
+        if self._card is None:
+            if SEARCH_CARD_TEST_ID_RE.fullmatch(test_id):
+                self._card = {"id": None, "title": [], "subtitle": []}
+                self._card_depth = 0 if is_void else 1
+            return
+
+        if not is_void:
+            self._card_depth += 1
+
+        if tag.lower() == "a" and self._card.get("id") is None:
+            self._card["id"] = self._listing_id_from_href(attributes.get("href"))
+
+        if test_id.endswith("-title"):
+            self._title_container_depth = self._card_depth
+        if test_id == "listing-title-card-view":
+            self._title_text_depth += 1
+
+        title_attribute = (attributes.get("title") or "").strip()
+        if title_attribute:
+            if test_id == "listing-title-card-view":
+                self._card["title"].append(title_attribute)
+            elif self._title_container_depth is not None:
+                self._card["subtitle"].append(title_attribute)
+
+    def handle_data(self, data: str) -> None:
+        if self._card is not None and self._title_text_depth:
+            text = data.strip()
+            if text:
+                self._card["title"].append(text)
+
+    def handle_endtag(self, tag: str) -> None:
+        if self._card is None or tag.lower() in VOID_HTML_TAGS:
+            return
+        if self._title_text_depth and tag.lower() == "span":
+            self._title_text_depth -= 1
+        if self._title_container_depth == self._card_depth:
+            self._title_container_depth = None
+        self._card_depth -= 1
+        if self._card_depth != 0:
+            return
+
+        listing_id = self._card.get("id")
+        title = " ".join(dict.fromkeys(self._card["title"])).strip()
+        subtitle = " ".join(dict.fromkeys(self._card["subtitle"])).strip()
+        if listing_id and (title or subtitle):
+            self.enrichment_by_id[str(listing_id)] = {
+                "searchSsrTitle": title,
+                "searchSsrSubtitle": subtitle,
+                "searchSsrDriveText": " ".join(part for part in (title, subtitle) if part),
+            }
+        self._card = None
+        self._title_container_depth = None
+        self._title_text_depth = 0
+
+
+def extract_search_card_enrichment(html: str) -> dict[str, dict[str, str]]:
+    """Return SSR title/subtitle enrichment keyed by the Search listing ID.
+
+    This reads only fields rendered inside the card's title block; it never uses
+    generic dealer, location, or arbitrary card text as a vehicle attribute.
+    """
+    parser = _SearchCardEnrichmentParser()
+    parser.feed(html)
+    parser.close()
+    return parser.enrichment_by_id
 
 
 def extract_next_flight_strings(html: str) -> list[str]:
@@ -77,4 +172,51 @@ def extract_detail_listing(html: str) -> dict[str, Any]:
         listing = extract_json_object_after(chunk, '"listing":')
         if isinstance(listing, dict):
             return listing
+    return extract_legacy_detail_listing(html)
+
+
+def extract_legacy_detail_listing(html: str) -> dict[str, Any]:
+    """Extract and normalize the compact legacy VIP INITIAL_STATE payload."""
+    state = extract_json_object_after(html, "window.__INITIAL_STATE__ =")
+    if not isinstance(state, dict):
+        return {}
+
+    search = state.get("search")
+    vip = search.get("vip") if isinstance(search, dict) else None
+    ads = vip.get("ads") if isinstance(vip, dict) else None
+    if not isinstance(ads, dict):
+        return {}
+
+    for payload in ads.values():
+        data = payload.get("data") if isinstance(payload, dict) else None
+        ad = data.get("ad") if isinstance(data, dict) else None
+        if not isinstance(ad, dict):
+            continue
+
+        listing = dict(ad)
+        make = listing.get("make")
+        if isinstance(make, str):
+            listing["make"] = {"localized": make}
+        model = listing.get("model")
+        if isinstance(model, str):
+            listing["model"] = {"localized": model}
+
+        contact_info = listing.get("contactInfo")
+        if "contact" not in listing and isinstance(contact_info, dict):
+            listing["contact"] = contact_info
+
+        gallery_images = listing.get("galleryImages")
+        if "images" not in listing and isinstance(gallery_images, list):
+            listing["images"] = gallery_images
+
+        price = listing.get("price")
+        if isinstance(price, dict) and not isinstance(price.get("grs"), dict):
+            amount = price.get("grossAmount")
+            currency = price.get("grossCurrency")
+            if amount is not None or currency is not None:
+                normalized_price = dict(price)
+                normalized_price["grs"] = {"amount": amount, "currency": currency}
+                listing["price"] = normalized_price
+
+        return listing
     return {}

mobilede_scraper/mobile_de/mapper.py

@@ -29,39 +29,6 @@ _RU_BODY_MAP = {
     "малолитражный": "HATCHBACK",
 }
 
-_LEGACY_BODY_MAP = {
-    "cabrio": "OPEN",
-    "cabriolet": "OPEN",
-    "roadster": "OPEN",
-    "limousine": "SEDAN",
-    "saloon": "SEDAN",
-    "sedan": "SEDAN",
-    "suv": "SUV",
-    "offroad": "SUV",
-    "gelandewagen": "SUV",
-    "geländewagen": "SUV",
-    "pickup": "PICKUP",
-    "pick-up": "PICKUP",
-    "kombi": "STATION_WAGON",
-    "estatecar": "STATION_WAGON",
-    "touring": "STATION_WAGON",
-    "estate": "STATION_WAGON",
-    "van": "MINIVAN",
-    "kleinbus": "MINIVAN",
-    "bus": "MINIVAN",
-    "active tourer": "MINIVAN",
-    "gran tourer": "MINIVAN",
-    "coupe": "COUPE",
-    "sportscar": "COUPE",
-    "sports car": "COUPE",
-    "hatchback": "HATCHBACK",
-    "kleinwagen": "HATCHBACK",
-    "smallcar": "HATCHBACK",
-    "small car": "HATCHBACK",
-    "compact": "HATCHBACK",
-    "compactcar": "HATCHBACK",
-}
-
 _RU_GEARBOX_MAP = {
     "автоматическая": "AT",
     "автоматическая коробка": "AT",
@@ -71,13 +38,6 @@ _RU_GEARBOX_MAP = {
     "вариатор": "CVT",
 }
 
-_LEGACY_GEARBOX_MAP = {
-    "automatik": "AT",
-    "automatic": "AT",
-    "manual": "MT",
-    "cvt": "CVT",
-}
-
 _RU_COLOR_MAP = {
     "черный": "black",
     "чёрный": "black",
@@ -101,65 +61,6 @@ _RU_COLOR_MAP = {
     "фиолетовый": "purple",
 }
 
-_LEGACY_COLOR_MAP = {
-    "schwarz": "black",
-    "saphirschwarz": "black",
-    "carbonschwarz": "black",
-    "obsidianschwarz": "black",
-    "jet black": "black",
-    "jetblack": "black",
-    "black": "black",
-    "weiss": "white",
-    "weiß": "white",
-    "alpinweiss": "white",
-    "alpine white": "white",
-    "mineralweiss": "white",
-    "white": "white",
-    "silber": "silver",
-    "argent": "silver",
-    "silver": "silver",
-    "grau": "gray",
-    "grey": "gray",
-    "anthrazit": "gray",
-    "anthracite": "gray",
-    "graphit": "gray",
-    "graphite": "gray",
-    "spacegrau": "gray",
-    "brooklyn grau": "gray",
-    "brooklyn grey": "gray",
-    "sophistograu": "gray",
-    "skyscraper grau": "gray",
-    "gray": "gray",
-    "rot": "red",
-    "burgundy": "red",
-    "bordeaux": "red",
-    "maroon": "red",
-    "red": "red",
-    "blau": "blue",
-    "turquoise": "blue",
-    "cyan": "blue",
-    "blue": "blue",
-    "grün": "green",
-    "gruen": "green",
-    "green": "green",
-    "braun": "brown",
-    "brown": "brown",
-    "beige": "beige",
-    "champagner": "beige",
-    "champagne": "beige",
-    "creme": "beige",
-    "cream": "beige",
-    "ivory": "beige",
-    "gelb": "yellow",
-    "yellow": "yellow",
-    "orange": "orange",
-    "gold": "gold",
-    "bronze": "bronze",
-    "violett": "purple",
-    "lila": "purple",
-    "purple": "purple",
-}
-
 _RU_DRIVE_MARKERS = {
     "FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
     "RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
@@ -167,12 +68,12 @@ _RU_DRIVE_MARKERS = {
     "2WD": ("привод на два колеса", "два ведущих колеса"),
 }
 
-_LEGACY_DRIVE_MARKERS = {
-    "FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"),
-    "RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"),
-    "4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"),
-    "2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"),
-}
+_SEARCH_SSR_DRIVE_MARKERS = (
+    ("4WD", re.compile(r"\b(?:xdrive(?:\b|\d)|quattro|4matic|4motion|all4|awd|4wd|4x4|allrad(?:antrieb)?|carrera\s*4s?)\b", re.IGNORECASE)),
+    ("FWD", re.compile(r"\bfrontantrieb\b", re.IGNORECASE)),
+    ("RWD", re.compile(r"\bheckantrieb\b", re.IGNORECASE)),
+    ("2WD", re.compile(r"\bsdrive(?:\b|\d)", re.IGNORECASE)),
+)
 
 _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
 _MOBILEDE_IMAGE_HOST = "img.classistatic.de"
@@ -226,21 +127,16 @@ class MobileDeMapper:
         drive_text = " ".join(
             part
             for part in [
-                listing.title,
-                listing.subtitle,
-                self._mapping_value(attr, "an"),
                 attr_drive,
                 raw.get("wheelDrive"),
                 raw.get("drivetrain"),
                 raw.get("drive"),
                 raw.get("driveType"),
-                raw.get("modelDescription"),
-                raw.get("variant"),
-                raw.get("trim"),
                 nested_drive,
             ]
             if isinstance(part, str) and part.strip()
         )
+        search_ssr_drive = self._normalize_search_ssr_drive(raw.get("searchSsrDriveText"))
         damage_text = self._text(
             raw.get("damageCondition")
             or attr.get("damageCondition")
@@ -289,7 +185,7 @@ class MobileDeMapper:
                     nested_color,
                 )
             ),
-            drive=self._normalize_drive(drive_text),
+            drive=self._normalize_drive(drive_text) or search_ssr_drive,
             gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
             steering_wheel=self._normalize_steering(
                 steering_text,
@@ -381,11 +277,10 @@ class MobileDeMapper:
                 detail.get("drive"),
                 detail.get("driveType"),
                 self._find_first_value(detail, _DRIVE_VALUE_KEYS),
-                feature_text,
             ]
             if isinstance(part, str) and part.strip()
         )
-        drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}")
+        drive = self._normalize_drive(explicit_drive_text)
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
         is_damaged = self._is_damaged_value(
@@ -455,14 +350,12 @@ class MobileDeMapper:
                 "прокатный автомобиль",
                 "арендный автомобиль",
                 "rental vehicle",
-                "mietwagen",
             ),
             repair_history=self._contains_marker(
                 damage_text,
                 "устраненные повреждения",
                 "устранённые повреждения",
                 "repaired damage",
-                "reparierter unfallschaden",
             ),
             slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
@@ -574,36 +467,43 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_gearbox(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP):
-            for marker, mapped in value_map.items():
-                if marker in text:
-                    return mapped
+        for marker, mapped in _RU_GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
         return None
 
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS):
-            for mapped, markers in marker_map.items():
-                if any(marker in text for marker in markers):
-                    return mapped
+        for mapped, markers in _RU_DRIVE_MARKERS.items():
+            if any(marker in text for marker in markers):
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_search_ssr_drive(value: Any) -> str | None:
+        """Map only explicit, high-confidence drive tokens rendered by Search SSR."""
+        text = "" if value is None else str(value)
+        for mapped, pattern in _SEARCH_SSR_DRIVE_MARKERS:
+            if pattern.search(text):
+                return mapped
         return None
 
     @classmethod
-    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str:
-        if right_hand_drive is not None:
-            return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT"
-        text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")):
+    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str | None:
+        if right_hand_drive is not None and cls._bool_value(right_hand_drive):
             return "RIGHT"
-        if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")):
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("правый руль", "праворуль")):
+            return "RIGHT"
+        if any(marker in text for marker in ("левый руль", "леворуль")):
             return "LEFT"
-        return "LEFT"
+        return None
 
     @staticmethod
     def _normalize_country(value: Any) -> str:
         text = "" if value is None else str(value).strip().upper()
-        if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}:
+        if text in {"DE", "ГЕРМАНИЯ"}:
             return "DE"
         if text in {"US", "USA", "UNITED STATES"}:
             return "US"
@@ -624,13 +524,9 @@ class MobileDeMapper:
         text = "" if value is None else str(value).lower()
         if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
             return "SUV"
-        for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP):
-            for marker, mapped in value_map.items():
-                if marker in {"van", "bus"}:
-                    if re.search(rf"\b{re.escape(marker)}\b", text):
-                        return mapped
-                elif marker in text:
-                    return mapped
+        for marker, mapped in _RU_BODY_MAP.items():
+            if marker in text:
+                return mapped
         return "OTHER"
 
     @staticmethod
@@ -709,14 +605,10 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_color(value: Any) -> str:
         text = "" if value is None else str(value).lower().strip()
-        text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
         text = re.sub(r"[_\-/]+", " ", text)
-        if text == "wei":
-            return "white"
-        for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP):
-            for marker, mapped in value_map.items():
-                if marker in text:
-                    return mapped
+        for marker, mapped in _RU_COLOR_MAP.items():
+            if marker in text:
+                return mapped
         return "other"
 
     @staticmethod
@@ -830,6 +722,10 @@ class MobileDeMapper:
             return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
         return MobileDeMapper._images_from_listing(raw)
 
+    def detail_to_images(self, raw: dict[str, Any]) -> list[ImageRecord]:
+        """Extract the complete gallery without mapping Detail car fields."""
+        return self._images_from_detail(raw)
+
     @staticmethod
     def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
         if depth > 8: