From f3329f76a57306ad18c2d955e13d3a1f0692ac1c Mon Sep 17 00:00:00 2001 From: qananasikq Date: Tue, 18 Aug 2026 15:59:00 +0300 Subject: [PATCH] Normalize Russian vehicle data --- mobilede_scraper/mobile_de/mapper.py | 217 +++++++++++++++++++-------- tests/test_mappers.py | 114 ++++++++++++++ 2 files changed, 267 insertions(+), 64 deletions(-) diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py index b5939dc..c382064 100644 --- a/mobilede_scraper/mobile_de/mapper.py +++ b/mobilede_scraper/mobile_de/mapper.py @@ -14,43 +14,46 @@ from .models import MobileDeListing PARSER_ID_ALPHABET = ascii_letters + digits PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$") -_BODY_MAP = { +_RU_BODY_MAP = { "кабриолет": "OPEN", "родстер": "OPEN", + "седан": "SEDAN", + "внедорожник": "SUV", + "пикап": "PICKUP", + "универсал": "STATION_WAGON", + "минивэн": "MINIVAN", + "микроавтобус": "MINIVAN", + "фургон": "MINIVAN", + "купе": "COUPE", + "хэтчбек": "HATCHBACK", + "малолитражный": "HATCHBACK", +} + +_LEGACY_BODY_MAP = { "cabrio": "OPEN", "cabriolet": "OPEN", "roadster": "OPEN", - "седан": "SEDAN", "limousine": "SEDAN", "saloon": "SEDAN", "sedan": "SEDAN", - "внедорожник": "SUV", "suv": "SUV", "offroad": "SUV", "gelandewagen": "SUV", "geländewagen": "SUV", - "пикап": "PICKUP", "pickup": "PICKUP", "pick-up": "PICKUP", - "универсал": "STATION_WAGON", "kombi": "STATION_WAGON", "estatecar": "STATION_WAGON", "touring": "STATION_WAGON", "estate": "STATION_WAGON", - "минивэн": "MINIVAN", - "микроавтобус": "MINIVAN", - "фургон": "MINIVAN", "van": "MINIVAN", "kleinbus": "MINIVAN", "bus": "MINIVAN", "active tourer": "MINIVAN", "gran tourer": "MINIVAN", - "купе": "COUPE", "coupe": "COUPE", "sportscar": "COUPE", "sports car": "COUPE", - "хэтчбек": "HATCHBACK", - "малолитражный": "HATCHBACK", "hatchback": "HATCHBACK", "kleinwagen": "HATCHBACK", "smallcar": "HATCHBACK", @@ -59,21 +62,46 @@ _BODY_MAP = { "compactcar": "HATCHBACK", } -_GEARBOX_MAP = { +_RU_GEARBOX_MAP = { "автоматическая": "AT", "автоматическая коробка": "AT", - "automatik": "AT", - "automatic": "AT", + "механика": "MT", "механическая": "MT", "механическая коробка": "MT", - "manual": "MT", "вариатор": "CVT", +} + +_LEGACY_GEARBOX_MAP = { + "automatik": "AT", + "automatic": "AT", + "manual": "MT", "cvt": "CVT", } -_COLOR_MAP = { +_RU_COLOR_MAP = { "черный": "black", "чёрный": "black", + "белый": "white", + "серебристый": "silver", + "серебряный": "silver", + "серый": "gray", + "красный": "red", + "синий": "blue", + "голубой": "blue", + "зеленый": "green", + "зелёный": "green", + "коричневый": "brown", + "бежевый": "beige", + "желтый": "yellow", + "жёлтый": "yellow", + "оранжевый": "orange", + "золотистый": "gold", + "золотой": "gold", + "бронзовый": "bronze", + "фиолетовый": "purple", +} + +_LEGACY_COLOR_MAP = { "schwarz": "black", "saphirschwarz": "black", "carbonschwarz": "black", @@ -81,19 +109,15 @@ _COLOR_MAP = { "jet black": "black", "jetblack": "black", "black": "black", - "белый": "white", "weiss": "white", "weiß": "white", "alpinweiss": "white", "alpine white": "white", "mineralweiss": "white", "white": "white", - "серебристый": "silver", - "серебряный": "silver", "silber": "silver", "argent": "silver", "silver": "silver", - "серый": "gray", "grau": "gray", "grey": "gray", "anthrazit": "gray", @@ -106,49 +130,50 @@ _COLOR_MAP = { "sophistograu": "gray", "skyscraper grau": "gray", "gray": "gray", - "красный": "red", "rot": "red", "burgundy": "red", "bordeaux": "red", "maroon": "red", "red": "red", - "синий": "blue", - "голубой": "blue", "blau": "blue", "turquoise": "blue", "cyan": "blue", "blue": "blue", - "зеленый": "green", - "зелёный": "green", "grün": "green", "gruen": "green", "green": "green", - "коричневый": "brown", "braun": "brown", "brown": "brown", - "бежевый": "beige", "beige": "beige", "champagner": "beige", "champagne": "beige", "creme": "beige", "cream": "beige", "ivory": "beige", - "желтый": "yellow", - "жёлтый": "yellow", "gelb": "yellow", "yellow": "yellow", - "оранжевый": "orange", "orange": "orange", - "золотой": "gold", "gold": "gold", - "бронзовый": "bronze", "bronze": "bronze", - "фиолетовый": "purple", "violett": "purple", "lila": "purple", "purple": "purple", } +_RU_DRIVE_MARKERS = { + "FWD": ("передний привод", "привод на передние колеса", "передние колеса"), + "RWD": ("задний привод", "привод на задние колеса", "задние колеса"), + "4WD": ("полный привод", "привод на четыре колеса", "все колеса"), + "2WD": ("привод на два колеса", "два ведущих колеса"), +} + +_LEGACY_DRIVE_MARKERS = { + "FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"), + "RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"), + "4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"), + "2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"), +} + _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"] _MOBILEDE_IMAGE_HOST = "img.classistatic.de" _MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/" @@ -161,6 +186,7 @@ _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"} _ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"} _DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"} _GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"} +_STEERING_VALUE_KEYS = {"steeringwheel", "steeringposition", "steering", "righthanddrive"} class MobileDeMapper: @@ -183,12 +209,20 @@ class MobileDeMapper: nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS) nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS) nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS) + nested_steering = self._find_first_value(raw, _STEERING_VALUE_KEYS) attr_country = self._mapping_value(attr, "cn", "countryCode", "country") attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint") attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body") attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement") attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart") attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType") + attr_steering = self._mapping_value(attr, "steeringWheel", "steeringPosition", "steering", "rightHandDrive") + raw_features = raw.get("features") if isinstance(raw.get("features"), list) else [] + steering_text = " ".join( + str(part) + for part in [attr_steering, nested_steering, *raw_features] + if part is not None and str(part).strip() + ) drive_text = " ".join( part for part in [ @@ -257,7 +291,10 @@ class MobileDeMapper: ), drive=self._normalize_drive(drive_text), gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox), - steering_wheel="LEFT", + steering_wheel=self._normalize_steering( + steering_text, + right_hand_drive=self._first_present(raw.get("rightHandDrive"), attr.get("rightHandDrive")), + ), body_type=self._normalize_body_from_candidates( attr_body, raw.get("category"), @@ -281,7 +318,7 @@ class MobileDeMapper: ), selling_type="STOCK", one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")), - new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")), + new_car=self._bool_value(raw.get("isConditionNew")), is_hidden=False, origin="MOBILEDE", origin_url=listing.url, @@ -333,23 +370,28 @@ class MobileDeMapper: self._find_first_value(detail, _COLOR_VALUE_KEYS), ) ) - drive_text = " ".join( + features = detail.get("features") if isinstance(detail.get("features"), list) else [] + feature_text = " ".join(str(feature) for feature in features if isinstance(feature, str)) + explicit_drive_text = " ".join( part for part in [ - short_title, - subtitle, self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"), detail.get("wheelDrive"), detail.get("drivetrain"), detail.get("drive"), detail.get("driveType"), self._find_first_value(detail, _DRIVE_VALUE_KEYS), + feature_text, ] if isinstance(part, str) and part.strip() ) + drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}") damage_text = self._text(attrs.get("damageCondition")).lower() - is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text) + is_damaged = self._is_damaged_value( + self._first_present(detail.get("isDamageCase"), detail.get("hasDamage")), + damage_text, + ) owners_text = self._text( self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners") @@ -367,9 +409,25 @@ class MobileDeMapper: country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"), is_sold=False, color=color, - drive=self._normalize_drive(drive_text), + drive=drive, gearbox=gearbox, - steering_wheel="LEFT", + steering_wheel=self._normalize_steering( + " ".join( + str(part) + for part in [ + self._mapping_value(attrs, "steeringWheel", "steeringPosition", "steering"), + detail.get("steeringWheel"), + detail.get("steeringPosition"), + detail.get("steering"), + feature_text, + ] + if part is not None and str(part).strip() + ), + right_hand_drive=self._first_present( + detail.get("rightHandDrive"), + self._mapping_value(attrs, "rightHandDrive"), + ), + ), body_type=body_type, engine_volume=self._engine_volume_from_candidates( self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"), @@ -384,16 +442,28 @@ class MobileDeMapper: ), selling_type="STOCK", one_owner=one_owner, - new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")), + new_car=self._bool_value(detail.get("isConditionNew")), is_hidden=False, origin="MOBILEDE", origin_url=MobileDeClient.build_detail_url(listing_id), origin_id=origin_id, is_damaged=is_damaged, evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")), - non_smoking=False, - rental=False, - repair_history=False, + non_smoking=self._contains_marker(feature_text, "машина некурящего водителя"), + rental=self._contains_marker( + feature_text, + "прокатный автомобиль", + "арендный автомобиль", + "rental vehicle", + "mietwagen", + ), + repair_history=self._contains_marker( + damage_text, + "устраненные повреждения", + "устранённые повреждения", + "repaired damage", + "reparierter unfallschaden", + ), slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)), last_seen_at=datetime.now(timezone.utc), details_confirmed=True, @@ -504,22 +574,32 @@ class MobileDeMapper: @staticmethod def _normalize_gearbox(value: Any) -> str | None: text = "" if value is None else str(value).lower() - for marker, mapped in _GEARBOX_MAP.items(): - if marker in text: - return mapped + for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP): + for marker, mapped in value_map.items(): + if marker in text: + return mapped return None @staticmethod def _normalize_drive(value: Any) -> str | None: text = "" if value is None else str(value).lower() - if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")): - return "FWD" - if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")): - return "RWD" - if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")): - return "4WD" + for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS): + for mapped, markers in marker_map.items(): + if any(marker in text for marker in markers): + return mapped return None + @classmethod + def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str: + if right_hand_drive is not None: + return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT" + text = "" if value is None else str(value).lower() + if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")): + return "RIGHT" + if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")): + return "LEFT" + return "LEFT" + @staticmethod def _normalize_country(value: Any) -> str: text = "" if value is None else str(value).strip().upper() @@ -542,16 +622,22 @@ class MobileDeMapper: @staticmethod def _normalize_body(value: Any) -> str: text = "" if value is None else str(value).lower() - if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text): + if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text): return "SUV" - for marker, mapped in _BODY_MAP.items(): - if marker in {"van", "bus"}: - if re.search(rf"\b{re.escape(marker)}\b", text): + for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP): + for marker, mapped in value_map.items(): + if marker in {"van", "bus"}: + if re.search(rf"\b{re.escape(marker)}\b", text): + return mapped + elif marker in text: return mapped - elif marker in text: - return mapped return "OTHER" + @staticmethod + def _contains_marker(value: Any, *markers: str) -> bool: + text = "" if value is None else str(value).lower() + return any(marker.lower() in text for marker in markers) + @classmethod def _normalize_body_from_candidates(cls, *values: Any) -> str: for value in values: @@ -625,10 +711,13 @@ class MobileDeMapper: text = "" if value is None else str(value).lower().strip() text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss") text = re.sub(r"[_\-/]+", " ", text) - for marker, mapped in _COLOR_MAP.items(): - if marker in text: - return mapped - return text[:50] if text else "other" + if text == "wei": + return "white" + for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP): + for marker, mapped in value_map.items(): + if marker in text: + return mapped + return "other" @staticmethod def _is_one_owner(value: Any) -> bool: diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 243e019..5e2e30f 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -434,6 +434,17 @@ class TestMobileDeMapper(unittest.TestCase): self.assertEqual(record.drive, "FWD") + def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None: + record = self.mapper.detail_to_car_record( + "1014", + { + "shortTitle": "BMW X1", + "subTitle": "sDrive18i Advantage", + }, + ) + + self.assertEqual(record.drive, "2WD") + def test_russian_search_values_map_to_english_contract(self) -> None: record = self.mapper.listing_to_car_record( MobileDeListing( @@ -469,6 +480,64 @@ class TestMobileDeMapper(unittest.TestCase): self.assertFalse(record.is_damaged) self.assertFalse(record.new_car) + def test_russian_gold_color_maps_to_english_contract(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="2002", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002", + title="BMW X2", + raw={"attr": {"ecol": "Золотистый Металлик Матовый"}}, + ) + ) + + self.assertEqual(record.color, "gold") + + def test_truncated_white_color_maps_to_english_contract(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="2003", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003", + title="BMW 320", + raw={"attr": {"ecol": "Wei"}}, + ) + ) + + self.assertEqual(record.color, "white") + + def test_unknown_color_maps_to_other(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="2004", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004", + title="BMW 320", + raw={"attr": {"ecol": "Неизвестное покрытие"}}, + ) + ) + + self.assertEqual(record.color, "other") + + def test_right_hand_drive_maps_from_russian_feature(self) -> None: + record = self.mapper.detail_to_car_record( + "2005", + { + "shortTitle": "BMW 320", + "features": ["Правый руль"], + }, + ) + + self.assertEqual(record.steering_wheel, "RIGHT") + + def test_right_hand_drive_maps_from_explicit_flag(self) -> None: + record = self.mapper.detail_to_car_record( + "2006", + { + "shortTitle": "BMW 320", + "rightHandDrive": True, + }, + ) + + self.assertEqual(record.steering_wheel, "RIGHT") + def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None: first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111" second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222" @@ -482,6 +551,11 @@ class TestMobileDeMapper(unittest.TestCase): "model": {"localized": "Macan"}, "price": {"grs": {"amount": 59900, "currency": "EUR"}}, "isConditionNew": "false", + "isDamageCase": True, + "features": [ + "Привод на четыре колеса", + "Машина некурящего водителя", + ], "attributes": [ {"tag": "firstRegistration", "value": "04/2022"}, {"tag": "mileage", "value": "149 300 км"}, @@ -503,8 +577,12 @@ class TestMobileDeMapper(unittest.TestCase): self.assertEqual(record.gearbox, "AT") self.assertEqual(record.body_type, "SUV") self.assertEqual(record.engine_volume, 2894) + self.assertEqual(record.drive, "4WD") self.assertTrue(record.one_owner) self.assertTrue(record.is_damaged) + self.assertTrue(record.non_smoking) + self.assertTrue(record.repair_history) + self.assertFalse(record.rental) self.assertFalse(record.new_car) self.assertEqual(len(record.images), 2) self.assertEqual( @@ -515,6 +593,42 @@ class TestMobileDeMapper(unittest.TestCase): ], ) + def test_detail_is_new_ad_does_not_mean_new_car(self) -> None: + record = self.mapper.detail_to_car_record( + "2003", + { + "shortTitle": "BMW X5", + "make": {"localized": "BMW"}, + "model": {"localized": "X5"}, + "isNew": True, + "isConditionNew": False, + "attributes": [ + {"tag": "firstRegistration", "value": "05/2023"}, + {"tag": "mileage", "value": "108 398 км"}, + {"tag": "damageCondition", "value": "Подержанный"}, + ], + }, + ) + + self.assertFalse(record.new_car) + + def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None: + record = self.mapper.detail_to_car_record( + "2004", + { + "shortTitle": "BMW iX3", + "make": {"localized": "BMW"}, + "model": {"localized": "iX3"}, + "attributes": [ + {"tag": "transmission", "value": "Механика"}, + {"tag": "category", "value": "Иное"}, + ], + }, + ) + + self.assertEqual(record.gearbox, "MT") + self.assertEqual(record.body_type, "SUV") + if __name__ == "__main__": unittest.main()