qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize Russian vehicle data

Browse Source
This commit is contained in:
qananasikq
2026-08-18 15:59:00 +03:00
parent a19a2009ed
commit f3329f76a5
2 changed files with 267 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/mapper.py
+145 -56
View File
@@ -14,43 +14,46 @@ from .models import MobileDeListing
PARSER_ID_ALPHABET = ascii_letters + digits
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
_BODY_MAP = {
_RU_BODY_MAP = {
"кабриолет": "OPEN",
"родстер": "OPEN",
"седан": "SEDAN",
"внедорожник": "SUV",
"пикап": "PICKUP",
"универсал": "STATION_WAGON",
"минивэн": "MINIVAN",
"микроавтобус": "MINIVAN",
"фургон": "MINIVAN",
"купе": "COUPE",
"хэтчбек": "HATCHBACK",
"малолитражный": "HATCHBACK",
}
_LEGACY_BODY_MAP = {
"cabrio": "OPEN",
"cabriolet": "OPEN",
"roadster": "OPEN",
"седан": "SEDAN",
"limousine": "SEDAN",
"saloon": "SEDAN",
"sedan": "SEDAN",
"внедорожник": "SUV",
"suv": "SUV",
"offroad": "SUV",
"gelandewagen": "SUV",
"geländewagen": "SUV",
"пикап": "PICKUP",
"pickup": "PICKUP",
"pick-up": "PICKUP",
"универсал": "STATION_WAGON",
"kombi": "STATION_WAGON",
"estatecar": "STATION_WAGON",
"touring": "STATION_WAGON",
"estate": "STATION_WAGON",
"минивэн": "MINIVAN",
"микроавтобус": "MINIVAN",
"фургон": "MINIVAN",
"van": "MINIVAN",
"kleinbus": "MINIVAN",
"bus": "MINIVAN",
"active tourer": "MINIVAN",
"gran tourer": "MINIVAN",
"купе": "COUPE",
"coupe": "COUPE",
"sportscar": "COUPE",
"sports car": "COUPE",
"хэтчбек": "HATCHBACK",
"малолитражный": "HATCHBACK",
"hatchback": "HATCHBACK",
"kleinwagen": "HATCHBACK",
"smallcar": "HATCHBACK",
@@ -59,21 +62,46 @@ _BODY_MAP = {
"compactcar": "HATCHBACK",
}
_GEARBOX_MAP = {
_RU_GEARBOX_MAP = {
"автоматическая": "AT",
"автоматическая коробка": "AT",
"automatik": "AT",
"automatic": "AT",
"механика": "MT",
"механическая": "MT",
"механическая коробка": "MT",
"manual": "MT",
"вариатор": "CVT",
}
_LEGACY_GEARBOX_MAP = {
"automatik": "AT",
"automatic": "AT",
"manual": "MT",
"cvt": "CVT",
}
_COLOR_MAP = {
_RU_COLOR_MAP = {
"черный": "black",
"чёрный": "black",
"белый": "white",
"серебристый": "silver",
"серебряный": "silver",
"серый": "gray",
"красный": "red",
"синий": "blue",
"голубой": "blue",
"зеленый": "green",
"зелёный": "green",
"коричневый": "brown",
"бежевый": "beige",
"желтый": "yellow",
"жёлтый": "yellow",
"оранжевый": "orange",
"золотистый": "gold",
"золотой": "gold",
"бронзовый": "bronze",
"фиолетовый": "purple",
}
_LEGACY_COLOR_MAP = {
"schwarz": "black",
"saphirschwarz": "black",
"carbonschwarz": "black",
@@ -81,19 +109,15 @@ _COLOR_MAP = {
"jet black": "black",
"jetblack": "black",
"black": "black",
"белый": "white",
"weiss": "white",
"weiß": "white",
"alpinweiss": "white",
"alpine white": "white",
"mineralweiss": "white",
"white": "white",
"серебристый": "silver",
"серебряный": "silver",
"silber": "silver",
"argent": "silver",
"silver": "silver",
"серый": "gray",
"grau": "gray",
"grey": "gray",
"anthrazit": "gray",
@@ -106,49 +130,50 @@ _COLOR_MAP = {
"sophistograu": "gray",
"skyscraper grau": "gray",
"gray": "gray",
"красный": "red",
"rot": "red",
"burgundy": "red",
"bordeaux": "red",
"maroon": "red",
"red": "red",
"синий": "blue",
"голубой": "blue",
"blau": "blue",
"turquoise": "blue",
"cyan": "blue",
"blue": "blue",
"зеленый": "green",
"зелёный": "green",
"grün": "green",
"gruen": "green",
"green": "green",
"коричневый": "brown",
"braun": "brown",
"brown": "brown",
"бежевый": "beige",
"beige": "beige",
"champagner": "beige",
"champagne": "beige",
"creme": "beige",
"cream": "beige",
"ivory": "beige",
"желтый": "yellow",
"жёлтый": "yellow",
"gelb": "yellow",
"yellow": "yellow",
"оранжевый": "orange",
"orange": "orange",
"золотой": "gold",
"gold": "gold",
"бронзовый": "bronze",
"bronze": "bronze",
"фиолетовый": "purple",
"violett": "purple",
"lila": "purple",
"purple": "purple",
}
_RU_DRIVE_MARKERS = {
"FWD": ("передний привод", "привод на передние колеса", "передние колеса"),
"RWD": ("задний привод", "привод на задние колеса", "задние колеса"),
"4WD": ("полный привод", "привод на четыре колеса", "все колеса"),
"2WD": ("привод на два колеса", "два ведущих колеса"),
}
_LEGACY_DRIVE_MARKERS = {
"FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"),
"RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"),
"4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"),
"2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"),
}
_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
_MOBILEDE_IMAGE_HOST = "img.classistatic.de"
_MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
@@ -161,6 +186,7 @@ _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
_STEERING_VALUE_KEYS = {"steeringwheel", "steeringposition", "steering", "righthanddrive"}
class MobileDeMapper:
@@ -183,12 +209,20 @@ class MobileDeMapper:
nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
nested_steering = self._find_first_value(raw, _STEERING_VALUE_KEYS)
attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
attr_steering = self._mapping_value(attr, "steeringWheel", "steeringPosition", "steering", "rightHandDrive")
raw_features = raw.get("features") if isinstance(raw.get("features"), list) else []
steering_text = " ".join(
str(part)
for part in [attr_steering, nested_steering, *raw_features]
if part is not None and str(part).strip()
)
drive_text = " ".join(
part
for part in [
@@ -257,7 +291,10 @@ class MobileDeMapper:
),
drive=self._normalize_drive(drive_text),
gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
steering_wheel="LEFT",
steering_wheel=self._normalize_steering(
steering_text,
right_hand_drive=self._first_present(raw.get("rightHandDrive"), attr.get("rightHandDrive")),
),
body_type=self._normalize_body_from_candidates(
attr_body,
raw.get("category"),
@@ -281,7 +318,7 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")),
new_car=self._bool_value(raw.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=listing.url,
@@ -333,23 +370,28 @@ class MobileDeMapper:
self._find_first_value(detail, _COLOR_VALUE_KEYS),
)
)
drive_text = " ".join(
features = detail.get("features") if isinstance(detail.get("features"), list) else []
feature_text = " ".join(str(feature) for feature in features if isinstance(feature, str))
explicit_drive_text = " ".join(
part
for part in [
short_title,
subtitle,
self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
detail.get("wheelDrive"),
detail.get("drivetrain"),
detail.get("drive"),
detail.get("driveType"),
self._find_first_value(detail, _DRIVE_VALUE_KEYS),
feature_text,
]
if isinstance(part, str) and part.strip()
)
drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}")
damage_text = self._text(attrs.get("damageCondition")).lower()
is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text)
is_damaged = self._is_damaged_value(
self._first_present(detail.get("isDamageCase"), detail.get("hasDamage")),
damage_text,
)
owners_text = self._text(
self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
@@ -367,9 +409,25 @@ class MobileDeMapper:
country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
is_sold=False,
color=color,
drive=self._normalize_drive(drive_text),
drive=drive,
gearbox=gearbox,
steering_wheel="LEFT",
steering_wheel=self._normalize_steering(
" ".join(
str(part)
for part in [
self._mapping_value(attrs, "steeringWheel", "steeringPosition", "steering"),
detail.get("steeringWheel"),
detail.get("steeringPosition"),
detail.get("steering"),
feature_text,
]
if part is not None and str(part).strip()
),
right_hand_drive=self._first_present(
detail.get("rightHandDrive"),
self._mapping_value(attrs, "rightHandDrive"),
),
),
body_type=body_type,
engine_volume=self._engine_volume_from_candidates(
self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
@@ -384,16 +442,28 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=one_owner,
new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")),
new_car=self._bool_value(detail.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=MobileDeClient.build_detail_url(listing_id),
origin_id=origin_id,
is_damaged=is_damaged,
evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
non_smoking=False,
rental=False,
repair_history=False,
non_smoking=self._contains_marker(feature_text, "машина некурящего водителя"),
rental=self._contains_marker(
feature_text,
"прокатный автомобиль",
"арендный автомобиль",
"rental vehicle",
"mietwagen",
),
repair_history=self._contains_marker(
damage_text,
"устраненные повреждения",
"устранённые повреждения",
"repaired damage",
"reparierter unfallschaden",
),
slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
last_seen_at=datetime.now(timezone.utc),
details_confirmed=True,
@@ -504,7 +574,8 @@ class MobileDeMapper:
@staticmethod
def _normalize_gearbox(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
for marker, mapped in _GEARBOX_MAP.items():
for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP):
for marker, mapped in value_map.items():
if marker in text:
return mapped
return None
@@ -512,14 +583,23 @@ class MobileDeMapper:
@staticmethod
def _normalize_drive(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
return "FWD"
if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
return "RWD"
if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
return "4WD"
for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS):
for mapped, markers in marker_map.items():
if any(marker in text for marker in markers):
return mapped
return None
@classmethod
def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str:
if right_hand_drive is not None:
return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT"
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")):
return "RIGHT"
if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")):
return "LEFT"
return "LEFT"
@staticmethod
def _normalize_country(value: Any) -> str:
text = "" if value is None else str(value).strip().upper()
@@ -542,9 +622,10 @@ class MobileDeMapper:
@staticmethod
def _normalize_body(value: Any) -> str:
text = "" if value is None else str(value).lower()
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text):
return "SUV"
for marker, mapped in _BODY_MAP.items():
for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP):
for marker, mapped in value_map.items():
if marker in {"van", "bus"}:
if re.search(rf"\b{re.escape(marker)}\b", text):
return mapped
@@ -552,6 +633,11 @@ class MobileDeMapper:
return mapped
return "OTHER"
@staticmethod
def _contains_marker(value: Any, *markers: str) -> bool:
text = "" if value is None else str(value).lower()
return any(marker.lower() in text for marker in markers)
@classmethod
def _normalize_body_from_candidates(cls, *values: Any) -> str:
for value in values:
@@ -625,10 +711,13 @@ class MobileDeMapper:
text = "" if value is None else str(value).lower().strip()
text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
text = re.sub(r"[_\-/]+", " ", text)
for marker, mapped in _COLOR_MAP.items():
if text == "wei":
return "white"
for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP):
for marker, mapped in value_map.items():
if marker in text:
return mapped
return text[:50] if text else "other"
return "other"
@staticmethod
def _is_one_owner(value: Any) -> bool:
tests/test_mappers.py
+114
View File
@@ -434,6 +434,17 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.drive, "FWD")
def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None:
record = self.mapper.detail_to_car_record(
"1014",
{
"shortTitle": "BMW X1",
"subTitle": "sDrive18i Advantage",
},
)
self.assertEqual(record.drive, "2WD")
def test_russian_search_values_map_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
@@ -469,6 +480,64 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertFalse(record.is_damaged)
self.assertFalse(record.new_car)
def test_russian_gold_color_maps_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2002",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002",
title="BMW X2",
raw={"attr": {"ecol": "Золотистый Металлик Матовый"}},
)
)
self.assertEqual(record.color, "gold")
def test_truncated_white_color_maps_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2003",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003",
title="BMW 320",
raw={"attr": {"ecol": "Wei"}},
)
)
self.assertEqual(record.color, "white")
def test_unknown_color_maps_to_other(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2004",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004",
title="BMW 320",
raw={"attr": {"ecol": "Неизвестное покрытие"}},
)
)
self.assertEqual(record.color, "other")
def test_right_hand_drive_maps_from_russian_feature(self) -> None:
record = self.mapper.detail_to_car_record(
"2005",
{
"shortTitle": "BMW 320",
"features": ["Правый руль"],
},
)
self.assertEqual(record.steering_wheel, "RIGHT")
def test_right_hand_drive_maps_from_explicit_flag(self) -> None:
record = self.mapper.detail_to_car_record(
"2006",
{
"shortTitle": "BMW 320",
"rightHandDrive": True,
},
)
self.assertEqual(record.steering_wheel, "RIGHT")
def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
@@ -482,6 +551,11 @@ class TestMobileDeMapper(unittest.TestCase):
"model": {"localized": "Macan"},
"price": {"grs": {"amount": 59900, "currency": "EUR"}},
"isConditionNew": "false",
"isDamageCase": True,
"features": [
"Привод на четыре колеса",
"Машина некурящего водителя",
],
"attributes": [
{"tag": "firstRegistration", "value": "04/2022"},
{"tag": "mileage", "value": "149 300 км"},
@@ -503,8 +577,12 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 2894)
self.assertEqual(record.drive, "4WD")
self.assertTrue(record.one_owner)
self.assertTrue(record.is_damaged)
self.assertTrue(record.non_smoking)
self.assertTrue(record.repair_history)
self.assertFalse(record.rental)
self.assertFalse(record.new_car)
self.assertEqual(len(record.images), 2)
self.assertEqual(
@@ -515,6 +593,42 @@ class TestMobileDeMapper(unittest.TestCase):
],
)
def test_detail_is_new_ad_does_not_mean_new_car(self) -> None:
record = self.mapper.detail_to_car_record(
"2003",
{
"shortTitle": "BMW X5",
"make": {"localized": "BMW"},
"model": {"localized": "X5"},
"isNew": True,
"isConditionNew": False,
"attributes": [
{"tag": "firstRegistration", "value": "05/2023"},
{"tag": "mileage", "value": "108 398 км"},
{"tag": "damageCondition", "value": "Подержанный"},
],
},
)
self.assertFalse(record.new_car)
def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None:
record = self.mapper.detail_to_car_record(
"2004",
{
"shortTitle": "BMW iX3",
"make": {"localized": "BMW"},
"model": {"localized": "iX3"},
"attributes": [
{"tag": "transmission", "value": "Механика"},
{"tag": "category", "value": "Иное"},
],
},
)
self.assertEqual(record.gearbox, "MT")
self.assertEqual(record.body_type, "SUV")
if __name__ == "__main__":
unittest.main()