Normalize Russian vehicle data

mobilede_scraper/mobile_de/mapper.py

@@ -14,43 +14,46 @@ from .models import MobileDeListing
 PARSER_ID_ALPHABET = ascii_letters + digits
 PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
 
-_BODY_MAP = {
+_RU_BODY_MAP = {
     "кабриолет": "OPEN",
     "родстер": "OPEN",
+    "седан": "SEDAN",
+    "внедорожник": "SUV",
+    "пикап": "PICKUP",
+    "универсал": "STATION_WAGON",
+    "минивэн": "MINIVAN",
+    "микроавтобус": "MINIVAN",
+    "фургон": "MINIVAN",
+    "купе": "COUPE",
+    "хэтчбек": "HATCHBACK",
+    "малолитражный": "HATCHBACK",
+}
+
+_LEGACY_BODY_MAP = {
     "cabrio": "OPEN",
     "cabriolet": "OPEN",
     "roadster": "OPEN",
-    "седан": "SEDAN",
     "limousine": "SEDAN",
     "saloon": "SEDAN",
     "sedan": "SEDAN",
-    "внедорожник": "SUV",
     "suv": "SUV",
     "offroad": "SUV",
     "gelandewagen": "SUV",
     "geländewagen": "SUV",
-    "пикап": "PICKUP",
     "pickup": "PICKUP",
     "pick-up": "PICKUP",
-    "универсал": "STATION_WAGON",
     "kombi": "STATION_WAGON",
     "estatecar": "STATION_WAGON",
     "touring": "STATION_WAGON",
     "estate": "STATION_WAGON",
-    "минивэн": "MINIVAN",
-    "микроавтобус": "MINIVAN",
-    "фургон": "MINIVAN",
     "van": "MINIVAN",
     "kleinbus": "MINIVAN",
     "bus": "MINIVAN",
     "active tourer": "MINIVAN",
     "gran tourer": "MINIVAN",
-    "купе": "COUPE",
     "coupe": "COUPE",
     "sportscar": "COUPE",
     "sports car": "COUPE",
-    "хэтчбек": "HATCHBACK",
-    "малолитражный": "HATCHBACK",
     "hatchback": "HATCHBACK",
     "kleinwagen": "HATCHBACK",
     "smallcar": "HATCHBACK",
@@ -59,21 +62,46 @@ _BODY_MAP = {
     "compactcar": "HATCHBACK",
 }
 
-_GEARBOX_MAP = {
+_RU_GEARBOX_MAP = {
     "автоматическая": "AT",
     "автоматическая коробка": "AT",
-    "automatik": "AT",
+    "механика": "MT",
-    "automatic": "AT",
     "механическая": "MT",
     "механическая коробка": "MT",
-    "manual": "MT",
     "вариатор": "CVT",
+}
+
+_LEGACY_GEARBOX_MAP = {
+    "automatik": "AT",
+    "automatic": "AT",
+    "manual": "MT",
     "cvt": "CVT",
 }
 
-_COLOR_MAP = {
+_RU_COLOR_MAP = {
     "черный": "black",
     "чёрный": "black",
+    "белый": "white",
+    "серебристый": "silver",
+    "серебряный": "silver",
+    "серый": "gray",
+    "красный": "red",
+    "синий": "blue",
+    "голубой": "blue",
+    "зеленый": "green",
+    "зелёный": "green",
+    "коричневый": "brown",
+    "бежевый": "beige",
+    "желтый": "yellow",
+    "жёлтый": "yellow",
+    "оранжевый": "orange",
+    "золотистый": "gold",
+    "золотой": "gold",
+    "бронзовый": "bronze",
+    "фиолетовый": "purple",
+}
+
+_LEGACY_COLOR_MAP = {
     "schwarz": "black",
     "saphirschwarz": "black",
     "carbonschwarz": "black",
@@ -81,19 +109,15 @@ _COLOR_MAP = {
     "jet black": "black",
     "jetblack": "black",
     "black": "black",
-    "белый": "white",
     "weiss": "white",
     "weiß": "white",
     "alpinweiss": "white",
     "alpine white": "white",
     "mineralweiss": "white",
     "white": "white",
-    "серебристый": "silver",
-    "серебряный": "silver",
     "silber": "silver",
     "argent": "silver",
     "silver": "silver",
-    "серый": "gray",
     "grau": "gray",
     "grey": "gray",
     "anthrazit": "gray",
@@ -106,49 +130,50 @@ _COLOR_MAP = {
     "sophistograu": "gray",
     "skyscraper grau": "gray",
     "gray": "gray",
-    "красный": "red",
     "rot": "red",
     "burgundy": "red",
     "bordeaux": "red",
     "maroon": "red",
     "red": "red",
-    "синий": "blue",
-    "голубой": "blue",
     "blau": "blue",
     "turquoise": "blue",
     "cyan": "blue",
     "blue": "blue",
-    "зеленый": "green",
-    "зелёный": "green",
     "grün": "green",
     "gruen": "green",
     "green": "green",
-    "коричневый": "brown",
     "braun": "brown",
     "brown": "brown",
-    "бежевый": "beige",
     "beige": "beige",
     "champagner": "beige",
     "champagne": "beige",
     "creme": "beige",
     "cream": "beige",
     "ivory": "beige",
-    "желтый": "yellow",
-    "жёлтый": "yellow",
     "gelb": "yellow",
     "yellow": "yellow",
-    "оранжевый": "orange",
     "orange": "orange",
-    "золотой": "gold",
     "gold": "gold",
-    "бронзовый": "bronze",
     "bronze": "bronze",
-    "фиолетовый": "purple",
     "violett": "purple",
     "lila": "purple",
     "purple": "purple",
 }
 
+_RU_DRIVE_MARKERS = {
+    "FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
+    "RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
+    "4WD": ("полный привод", "привод на четыре колеса", "все колеса"),
+    "2WD": ("привод на два колеса", "два ведущих колеса"),
+}
+
+_LEGACY_DRIVE_MARKERS = {
+    "FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"),
+    "RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"),
+    "4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"),
+    "2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"),
+}
+
 _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
 _MOBILEDE_IMAGE_HOST = "img.classistatic.de"
 _MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
@@ -161,6 +186,7 @@ _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
 _ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
 _DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
 _GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
+_STEERING_VALUE_KEYS = {"steeringwheel", "steeringposition", "steering", "righthanddrive"}
 
 
 class MobileDeMapper:
@@ -183,12 +209,20 @@ class MobileDeMapper:
         nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
         nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
         nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
+        nested_steering = self._find_first_value(raw, _STEERING_VALUE_KEYS)
         attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
         attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
         attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
         attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
         attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
         attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
+        attr_steering = self._mapping_value(attr, "steeringWheel", "steeringPosition", "steering", "rightHandDrive")
+        raw_features = raw.get("features") if isinstance(raw.get("features"), list) else []
+        steering_text = " ".join(
+            str(part)
+            for part in [attr_steering, nested_steering, *raw_features]
+            if part is not None and str(part).strip()
+        )
         drive_text = " ".join(
             part
             for part in [
@@ -257,7 +291,10 @@ class MobileDeMapper:
             ),
             drive=self._normalize_drive(drive_text),
             gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
-            steering_wheel="LEFT",
+            steering_wheel=self._normalize_steering(
+                steering_text,
+                right_hand_drive=self._first_present(raw.get("rightHandDrive"), attr.get("rightHandDrive")),
+            ),
             body_type=self._normalize_body_from_candidates(
                 attr_body,
                 raw.get("category"),
@@ -281,7 +318,7 @@ class MobileDeMapper:
             ),
             selling_type="STOCK",
             one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
-            new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")),
+            new_car=self._bool_value(raw.get("isConditionNew")),
             is_hidden=False,
             origin="MOBILEDE",
             origin_url=listing.url,
@@ -333,23 +370,28 @@ class MobileDeMapper:
                 self._find_first_value(detail, _COLOR_VALUE_KEYS),
             )
         )
-        drive_text = " ".join(
+        features = detail.get("features") if isinstance(detail.get("features"), list) else []
+        feature_text = " ".join(str(feature) for feature in features if isinstance(feature, str))
+        explicit_drive_text = " ".join(
             part
             for part in [
-                short_title,
-                subtitle,
                 self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
                 detail.get("wheelDrive"),
                 detail.get("drivetrain"),
                 detail.get("drive"),
                 detail.get("driveType"),
                 self._find_first_value(detail, _DRIVE_VALUE_KEYS),
+                feature_text,
             ]
             if isinstance(part, str) and part.strip()
         )
+        drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}")
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
-        is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text)
+        is_damaged = self._is_damaged_value(
+            self._first_present(detail.get("isDamageCase"), detail.get("hasDamage")),
+            damage_text,
+        )
 
         owners_text = self._text(
             self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
@@ -367,9 +409,25 @@ class MobileDeMapper:
             country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
             is_sold=False,
             color=color,
-            drive=self._normalize_drive(drive_text),
+            drive=drive,
             gearbox=gearbox,
-            steering_wheel="LEFT",
+            steering_wheel=self._normalize_steering(
+                " ".join(
+                    str(part)
+                    for part in [
+                        self._mapping_value(attrs, "steeringWheel", "steeringPosition", "steering"),
+                        detail.get("steeringWheel"),
+                        detail.get("steeringPosition"),
+                        detail.get("steering"),
+                        feature_text,
+                    ]
+                    if part is not None and str(part).strip()
+                ),
+                right_hand_drive=self._first_present(
+                    detail.get("rightHandDrive"),
+                    self._mapping_value(attrs, "rightHandDrive"),
+                ),
+            ),
             body_type=body_type,
             engine_volume=self._engine_volume_from_candidates(
                 self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
@@ -384,16 +442,28 @@ class MobileDeMapper:
             ),
             selling_type="STOCK",
             one_owner=one_owner,
-            new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")),
+            new_car=self._bool_value(detail.get("isConditionNew")),
             is_hidden=False,
             origin="MOBILEDE",
             origin_url=MobileDeClient.build_detail_url(listing_id),
             origin_id=origin_id,
             is_damaged=is_damaged,
             evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
-            non_smoking=False,
+            non_smoking=self._contains_marker(feature_text, "машина некурящего водителя"),
-            rental=False,
+            rental=self._contains_marker(
-            repair_history=False,
+                feature_text,
+                "прокатный автомобиль",
+                "арендный автомобиль",
+                "rental vehicle",
+                "mietwagen",
+            ),
+            repair_history=self._contains_marker(
+                damage_text,
+                "устраненные повреждения",
+                "устранённые повреждения",
+                "repaired damage",
+                "reparierter unfallschaden",
+            ),
             slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
             last_seen_at=datetime.now(timezone.utc),
             details_confirmed=True,
@@ -504,7 +574,8 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_gearbox(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        for marker, mapped in _GEARBOX_MAP.items():
+        for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP):
+            for marker, mapped in value_map.items():
                 if marker in text:
                     return mapped
         return None
@@ -512,14 +583,23 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
+        for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS):
-            return "FWD"
+            for mapped, markers in marker_map.items():
-        if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
+                if any(marker in text for marker in markers):
-            return "RWD"
+                    return mapped
-        if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
-            return "4WD"
         return None
 
+    @classmethod
+    def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str:
+        if right_hand_drive is not None:
+            return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT"
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")):
+            return "RIGHT"
+        if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")):
+            return "LEFT"
+        return "LEFT"
+
     @staticmethod
     def _normalize_country(value: Any) -> str:
         text = "" if value is None else str(value).strip().upper()
@@ -542,9 +622,10 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_body(value: Any) -> str:
         text = "" if value is None else str(value).lower()
-        if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
+        if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
             return "SUV"
-        for marker, mapped in _BODY_MAP.items():
+        for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP):
+            for marker, mapped in value_map.items():
                 if marker in {"van", "bus"}:
                     if re.search(rf"\b{re.escape(marker)}\b", text):
                         return mapped
@@ -552,6 +633,11 @@ class MobileDeMapper:
                     return mapped
         return "OTHER"
 
+    @staticmethod
+    def _contains_marker(value: Any, *markers: str) -> bool:
+        text = "" if value is None else str(value).lower()
+        return any(marker.lower() in text for marker in markers)
+
     @classmethod
     def _normalize_body_from_candidates(cls, *values: Any) -> str:
         for value in values:
@@ -625,10 +711,13 @@ class MobileDeMapper:
         text = "" if value is None else str(value).lower().strip()
         text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
         text = re.sub(r"[_\-/]+", " ", text)
-        for marker, mapped in _COLOR_MAP.items():
+        if text == "wei":
+            return "white"
+        for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP):
+            for marker, mapped in value_map.items():
                 if marker in text:
                     return mapped
-        return text[:50] if text else "other"
+        return "other"
 
     @staticmethod
     def _is_one_owner(value: Any) -> bool:

tests/test_mappers.py

@@ -434,6 +434,17 @@ class TestMobileDeMapper(unittest.TestCase):
 
         self.assertEqual(record.drive, "FWD")
 
+    def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "1014",
+            {
+                "shortTitle": "BMW X1",
+                "subTitle": "sDrive18i Advantage",
+            },
+        )
+
+        self.assertEqual(record.drive, "2WD")
+
     def test_russian_search_values_map_to_english_contract(self) -> None:
         record = self.mapper.listing_to_car_record(
             MobileDeListing(
@@ -469,6 +480,64 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertFalse(record.is_damaged)
         self.assertFalse(record.new_car)
 
+    def test_russian_gold_color_maps_to_english_contract(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2002",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002",
+                title="BMW X2",
+                raw={"attr": {"ecol": "Золотистый Металлик Матовый"}},
+            )
+        )
+
+        self.assertEqual(record.color, "gold")
+
+    def test_truncated_white_color_maps_to_english_contract(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2003",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003",
+                title="BMW 320",
+                raw={"attr": {"ecol": "Wei"}},
+            )
+        )
+
+        self.assertEqual(record.color, "white")
+
+    def test_unknown_color_maps_to_other(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2004",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004",
+                title="BMW 320",
+                raw={"attr": {"ecol": "Неизвестное покрытие"}},
+            )
+        )
+
+        self.assertEqual(record.color, "other")
+
+    def test_right_hand_drive_maps_from_russian_feature(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2005",
+            {
+                "shortTitle": "BMW 320",
+                "features": ["Правый руль"],
+            },
+        )
+
+        self.assertEqual(record.steering_wheel, "RIGHT")
+
+    def test_right_hand_drive_maps_from_explicit_flag(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2006",
+            {
+                "shortTitle": "BMW 320",
+                "rightHandDrive": True,
+            },
+        )
+
+        self.assertEqual(record.steering_wheel, "RIGHT")
+
     def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
         first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
         second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
@@ -482,6 +551,11 @@ class TestMobileDeMapper(unittest.TestCase):
                 "model": {"localized": "Macan"},
                 "price": {"grs": {"amount": 59900, "currency": "EUR"}},
                 "isConditionNew": "false",
+                "isDamageCase": True,
+                "features": [
+                    "Привод на четыре колеса",
+                    "Машина некурящего водителя",
+                ],
                 "attributes": [
                     {"tag": "firstRegistration", "value": "04/2022"},
                     {"tag": "mileage", "value": "149 300 км"},
@@ -503,8 +577,12 @@ class TestMobileDeMapper(unittest.TestCase):
         self.assertEqual(record.gearbox, "AT")
         self.assertEqual(record.body_type, "SUV")
         self.assertEqual(record.engine_volume, 2894)
+        self.assertEqual(record.drive, "4WD")
         self.assertTrue(record.one_owner)
         self.assertTrue(record.is_damaged)
+        self.assertTrue(record.non_smoking)
+        self.assertTrue(record.repair_history)
+        self.assertFalse(record.rental)
         self.assertFalse(record.new_car)
         self.assertEqual(len(record.images), 2)
         self.assertEqual(
@@ -515,6 +593,42 @@ class TestMobileDeMapper(unittest.TestCase):
             ],
         )
 
+    def test_detail_is_new_ad_does_not_mean_new_car(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2003",
+            {
+                "shortTitle": "BMW X5",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "X5"},
+                "isNew": True,
+                "isConditionNew": False,
+                "attributes": [
+                    {"tag": "firstRegistration", "value": "05/2023"},
+                    {"tag": "mileage", "value": "108 398 км"},
+                    {"tag": "damageCondition", "value": "Подержанный"},
+                ],
+            },
+        )
+
+        self.assertFalse(record.new_car)
+
+    def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None:
+        record = self.mapper.detail_to_car_record(
+            "2004",
+            {
+                "shortTitle": "BMW iX3",
+                "make": {"localized": "BMW"},
+                "model": {"localized": "iX3"},
+                "attributes": [
+                    {"tag": "transmission", "value": "Механика"},
+                    {"tag": "category", "value": "Иное"},
+                ],
+            },
+        )
+
+        self.assertEqual(record.gearbox, "MT")
+        self.assertEqual(record.body_type, "SUV")
+
 
 if __name__ == "__main__":
     unittest.main()