Enrich search cards

mobilede_scraper/mobile_de/mapper.py

@@ -29,39 +29,6 @@ _RU_BODY_MAP = {
     "малолитражный": "HATCHBACK",
 }
 
-_LEGACY_BODY_MAP = {
-    "cabrio": "OPEN",
-    "cabriolet": "OPEN",
-    "roadster": "OPEN",
-    "limousine": "SEDAN",
-    "saloon": "SEDAN",
-    "sedan": "SEDAN",
-    "suv": "SUV",
-    "offroad": "SUV",
-    "gelandewagen": "SUV",
-    "geländewagen": "SUV",
-    "pickup": "PICKUP",
-    "pick-up": "PICKUP",
-    "kombi": "STATION_WAGON",
-    "estatecar": "STATION_WAGON",
-    "touring": "STATION_WAGON",
-    "estate": "STATION_WAGON",
-    "van": "MINIVAN",
-    "kleinbus": "MINIVAN",
-    "bus": "MINIVAN",
-    "active tourer": "MINIVAN",
-    "gran tourer": "MINIVAN",
-    "coupe": "COUPE",
-    "sportscar": "COUPE",
-    "sports car": "COUPE",
-    "hatchback": "HATCHBACK",
-    "kleinwagen": "HATCHBACK",
-    "smallcar": "HATCHBACK",
-    "small car": "HATCHBACK",
-    "compact": "HATCHBACK",
-    "compactcar": "HATCHBACK",
-}
-
 _RU_GEARBOX_MAP = {
     "автоматическая": "AT",
     "автоматическая коробка": "AT",
@@ -71,13 +38,6 @@ _RU_GEARBOX_MAP = {
     "вариатор": "CVT",
 }
 
-_LEGACY_GEARBOX_MAP = {
-    "automatik": "AT",
-    "automatic": "AT",
-    "manual": "MT",
-    "cvt": "CVT",
-}
-
 _RU_COLOR_MAP = {
     "черный": "black",
     "чёрный": "black",
@@ -101,65 +61,6 @@ _RU_COLOR_MAP = {
     "фиолетовый": "purple",
 }
 
-_LEGACY_COLOR_MAP = {
-    "schwarz": "black",
-    "saphirschwarz": "black",
-    "carbonschwarz": "black",
-    "obsidianschwarz": "black",
-    "jet black": "black",
-    "jetblack": "black",
-    "black": "black",
-    "weiss": "white",
-    "weiß": "white",
-    "alpinweiss": "white",
-    "alpine white": "white",
-    "mineralweiss": "white",
-    "white": "white",
-    "silber": "silver",
-    "argent": "silver",
-    "silver": "silver",
-    "grau": "gray",
-    "grey": "gray",
-    "anthrazit": "gray",
-    "anthracite": "gray",
-    "graphit": "gray",
-    "graphite": "gray",
-    "spacegrau": "gray",
-    "brooklyn grau": "gray",
-    "brooklyn grey": "gray",
-    "sophistograu": "gray",
-    "skyscraper grau": "gray",
-    "gray": "gray",
-    "rot": "red",
-    "burgundy": "red",
-    "bordeaux": "red",
-    "maroon": "red",
-    "red": "red",
-    "blau": "blue",
-    "turquoise": "blue",
-    "cyan": "blue",
-    "blue": "blue",
-    "grün": "green",
-    "gruen": "green",
-    "green": "green",
-    "braun": "brown",
-    "brown": "brown",
-    "beige": "beige",
-    "champagner": "beige",
-    "champagne": "beige",
-    "creme": "beige",
-    "cream": "beige",
-    "ivory": "beige",
-    "gelb": "yellow",
-    "yellow": "yellow",
-    "orange": "orange",
-    "gold": "gold",
-    "bronze": "bronze",
-    "violett": "purple",
-    "lila": "purple",
-    "purple": "purple",
-}
-
 _RU_DRIVE_MARKERS = {
     "FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
     "RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
@@ -167,12 +68,12 @@ _RU_DRIVE_MARKERS = {
     "2WD": ("привод на два колеса", "два ведущих колеса"),
 }
 
-_LEGACY_DRIVE_MARKERS = {
-    "FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"),
-    "RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"),
-    "4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"),
-    "2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"),
-}
+_SEARCH_SSR_DRIVE_MARKERS = (
+    ("4WD", re.compile(r"\b(?:xdrive(?:\b|\d)|quattro|4matic|4motion|all4|awd|4wd|4x4|allrad(?:antrieb)?|carrera\s*4s?)\b", re.IGNORECASE)),
+    ("FWD", re.compile(r"\bfrontantrieb\b", re.IGNORECASE)),
+    ("RWD", re.compile(r"\bheckantrieb\b", re.IGNORECASE)),
+    ("2WD", re.compile(r"\bsdrive(?:\b|\d)", re.IGNORECASE)),
+)
 
 _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
 _MOBILEDE_IMAGE_HOST = "img.classistatic.de"
@@ -226,21 +127,16 @@ class MobileDeMapper:
         drive_text = " ".join(
             part
             for part in [
-                listing.title,
-                listing.subtitle,
-                self._mapping_value(attr, "an"),
                 attr_drive,
                 raw.get("wheelDrive"),
                 raw.get("drivetrain"),
                 raw.get("drive"),
                 raw.get("driveType"),
-                raw.get("modelDescription"),
-                raw.get("variant"),
-                raw.get("trim"),
                 nested_drive,
             ]
             if isinstance(part, str) and part.strip()
         )
+        search_ssr_drive = self._normalize_search_ssr_drive(raw.get("searchSsrDriveText"))
         damage_text = self._text(
             raw.get("damageCondition")
             or attr.get("damageCondition")
@@ -289,7 +185,7 @@ class MobileDeMapper:
                     nested_color,
                 )
             ),
-            drive=self._normalize_drive(drive_text),
+            drive=self._normalize_drive(drive_text) or search_ssr_drive,
             gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
             steering_wheel=self._normalize_steering(
                 steering_text,
@@ -381,11 +277,10 @@ class MobileDeMapper:
                 detail.get("drive"),
                 detail.get("driveType"),
                 self._find_first_value(detail, _DRIVE_VALUE_KEYS),
-                feature_text,
             ]
             if isinstance(part, str) and part.strip()
         )
-        drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}")
+        drive = self._normalize_drive(explicit_drive_text)
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
         is_damaged = self._is_damaged_value(
@@ -455,14 +350,12 @@ class MobileDeMapper:
                 "прокатный автомобиль",
                 "арендный автомобиль",
                 "rental vehicle",
-                "mietwagen",
             ),
             repair_history=self._contains_marker(
                 damage_text,
                 "устраненные повреждения",
                 "устранённые повреждения",
                 "repaired damage",
-                "reparierter unfallschaden",
             ),
             slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
@@ -574,36 +467,43 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_gearbox(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP):
-            for marker, mapped in value_map.items():
-                if marker in text:
-                    return mapped
+        for marker, mapped in _RU_GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
         return None
 
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS):
-            for mapped, markers in marker_map.items():
-                if any(marker in text for marker in markers):
-                    return mapped
+        for mapped, markers in _RU_DRIVE_MARKERS.items():
+            if any(marker in text for marker in markers):
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_search_ssr_drive(value: Any) -> str | None:
+        """Map only explicit, high-confidence drive tokens rendered by Search SSR."""
+        text = "" if value is None else str(value)
+        for mapped, pattern in _SEARCH_SSR_DRIVE_MARKERS:
+            if pattern.search(text):
+                return mapped
         return None
 
     @classmethod
-    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str:
-        if right_hand_drive is not None:
-            return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT"
-        text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")):
+    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str | None:
+        if right_hand_drive is not None and cls._bool_value(right_hand_drive):
             return "RIGHT"
-        if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")):
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("правый руль", "праворуль")):
+            return "RIGHT"
+        if any(marker in text for marker in ("левый руль", "леворуль")):
             return "LEFT"
-        return "LEFT"
+        return None
 
     @staticmethod
     def _normalize_country(value: Any) -> str:
         text = "" if value is None else str(value).strip().upper()
-        if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}:
+        if text in {"DE", "ГЕРМАНИЯ"}:
             return "DE"
         if text in {"US", "USA", "UNITED STATES"}:
             return "US"
@@ -624,13 +524,9 @@ class MobileDeMapper:
         text = "" if value is None else str(value).lower()
         if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
             return "SUV"
-        for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP):
-            for marker, mapped in value_map.items():
-                if marker in {"van", "bus"}:
-                    if re.search(rf"\b{re.escape(marker)}\b", text):
-                        return mapped
-                elif marker in text:
-                    return mapped
+        for marker, mapped in _RU_BODY_MAP.items():
+            if marker in text:
+                return mapped
         return "OTHER"
 
     @staticmethod
@@ -709,14 +605,10 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_color(value: Any) -> str:
         text = "" if value is None else str(value).lower().strip()
-        text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
         text = re.sub(r"[_\-/]+", " ", text)
-        if text == "wei":
-            return "white"
-        for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP):
-            for marker, mapped in value_map.items():
-                if marker in text:
-                    return mapped
+        for marker, mapped in _RU_COLOR_MAP.items():
+            if marker in text:
+                return mapped
         return "other"
 
     @staticmethod
@@ -830,6 +722,10 @@ class MobileDeMapper:
             return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
         return MobileDeMapper._images_from_listing(raw)
 
+    def detail_to_images(self, raw: dict[str, Any]) -> list[ImageRecord]:
+        """Extract the complete gallery without mapping Detail car fields."""
+        return self._images_from_detail(raw)
+
     @staticmethod
     def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
         if depth > 8: