qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix parser identifiers and locale

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:30 +03:00
parent 2cebb78316
commit fba7299747
5 changed files with 185 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
mobilede_scraper/mobile_de/mapper.py
View File

@@ -3,6 +3,7 @@
import hashlib
import re
from datetime import datetime, timezone
from string import ascii_letters, digits
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -10,45 +11,36 @@ from ..storage.schemas import CarRecord, ImageRecord
from .client import MobileDeClient
from .models import MobileDeListing
PARSER_ID_ALPHABET = ascii_letters + digits
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
_BODY_MAP = {
"cabrio": "OPEN",
"cabriolet": "OPEN",
"кабриолет": "OPEN",
"roadster": "OPEN",
"кабрио": "OPEN",
"limousine": "SEDAN",
"saloon": "SEDAN",
"sedan": "SEDAN",
"седан": "SEDAN",
"suv": "SUV",
"offroad": "SUV",
"gelandewagen": "SUV",
"geländewagen": "SUV",
"pickup": "PICKUP",
"pick-up": "PICKUP",
"пикап": "PICKUP",
"внедорож": "SUV",
"kombi": "STATION_WAGON",
"estatecar": "STATION_WAGON",
"touring": "STATION_WAGON",
"estate": "STATION_WAGON",
"универсал": "STATION_WAGON",
"van": "MINIVAN",
"kleinbus": "MINIVAN",
"bus": "MINIVAN",
"active tourer": "MINIVAN",
"gran tourer": "MINIVAN",
"минивэн": "MINIVAN",
"фургон": "MINIVAN",
"coupe": "COUPE",
"sportscar": "COUPE",
"sports car": "COUPE",
"купе": "COUPE",
"hatchback": "HATCHBACK",
"kleinwagen": "HATCHBACK",
"кляйнваген": "HATCHBACK",
"малолитраж": "HATCHBACK",
"хэтч": "HATCHBACK",
"smallcar": "HATCHBACK",
"small car": "HATCHBACK",
"compact": "HATCHBACK",
@@ -56,10 +48,8 @@ _BODY_MAP = {
}
_GEARBOX_MAP = {
"автомат": "AT",
"automatik": "AT",
"automatic": "AT",
"механ": "MT",
"manual": "MT",
"cvt": "CVT",
}
@@ -71,18 +61,15 @@ _COLOR_MAP = {
"obsidianschwarz": "black",
"jet black": "black",
"jetblack": "black",
"черн": "black",
"black": "black",
"weiss": "white",
"weiß": "white",
"alpinweiss": "white",
"alpine white": "white",
"mineralweiss": "white",
"бел": "white",
"white": "white",
"silber": "silver",
"argent": "silver",
"сереб": "silver",
"silver": "silver",
"grau": "gray",
"grey": "gray",
@@ -95,25 +82,20 @@ _COLOR_MAP = {
"brooklyn grey": "gray",
"sophistograu": "gray",
"skyscraper grau": "gray",
"сер": "gray",
"gray": "gray",
"rot": "red",
"burgundy": "red",
"bordeaux": "red",
"maroon": "red",
"красн": "red",
"red": "red",
"blau": "blue",
"turquoise": "blue",
"cyan": "blue",
"син": "blue",
"blue": "blue",
"grün": "green",
"gruen": "green",
"зелен": "green",
"green": "green",
"braun": "brown",
"корич": "brown",
"brown": "brown",
"beige": "beige",
"champagner": "beige",
@@ -121,19 +103,13 @@ _COLOR_MAP = {
"creme": "beige",
"cream": "beige",
"ivory": "beige",
"беж": "beige",
"gelb": "yellow",
"желт": "yellow",
"yellow": "yellow",
"orange": "orange",
"оранж": "orange",
"gold": "gold",
"золот": "gold",
"bronze": "bronze",
"бронз": "bronze",
"violett": "purple",
"lila": "purple",
"фиолет": "purple",
"purple": "purple",
}
@@ -199,24 +175,24 @@ class MobileDeMapper:
).lower()
is_damaged = (
bool(raw.get("hasDamage"))
or ("дтп" in damage_text and "без дтп" not in damage_text)
or ("accident" in damage_text and "no accident" not in damage_text)
)
year = self._year_from_first_registration(
self._first_present(
listing.first_registration,
attr.get("fr"),
attr.get("yc"),
raw.get("firstRegistration"),
raw.get("firstRegistrationYear"),
raw.get("year"),
)
)
return CarRecord(
parser_id=self._parser_id(origin_id),
brand=brand[:50] or "UNKNOWN",
model=model[:50] or "UNKNOWN",
year=self._year_from_first_registration(
self._first_present(
listing.first_registration,
attr.get("fr"),
attr.get("yc"),
raw.get("firstRegistration"),
raw.get("firstRegistrationYear"),
raw.get("year"),
)
),
year=year,
price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
currency="EUR",
mileage=self._int_from_text(
@@ -279,7 +255,7 @@ class MobileDeMapper:
non_smoking=True,
rental=False,
repair_history=is_damaged,
slug=self._slugify(title or f"{brand} {model}"),
slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
last_seen_at=datetime.now(timezone.utc),
images=self._images_from_listing(raw),
)
@@ -334,7 +310,7 @@ class MobileDeMapper:
)
damage_text = self._text(attrs.get("damageCondition")).lower()
is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
owners_text = self._text(attrs.get("numPreviousOwners"))
one_owner = self._is_one_owner(owners_text)
@@ -377,7 +353,7 @@ class MobileDeMapper:
non_smoking=True,
rental=False,
repair_history=is_damaged,
slug=self._slugify(title or f"{brand} {model}"),
slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)),
last_seen_at=datetime.now(timezone.utc),
images=self._images_from_listing(detail),
)
@@ -388,8 +364,12 @@ class MobileDeMapper:
@staticmethod
def _parser_id(origin_id: str) -> str:
digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
return f"mobilede-{digest}"
number = int.from_bytes(hashlib.sha256(origin_id.encode("utf-8")).digest()[:17], "big")
chars: list[str] = []
for _ in range(22):
number, index = divmod(number, len(PARSER_ID_ALPHABET))
chars.append(PARSER_ID_ALPHABET[index])
return "car-" + "".join(chars)
@staticmethod
def _text(value: Any) -> str:
@@ -488,11 +468,11 @@ class MobileDeMapper:
@staticmethod
def _normalize_drive(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("front", "fwd", "перед", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
return "FWD"
if any(marker in text for marker in ("rear", "rwd", "зад", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
return "RWD"
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "полный", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
return "4WD"
return None
@@ -518,7 +498,6 @@ class MobileDeMapper:
@staticmethod
def _normalize_body(value: Any) -> str:
text = "" if value is None else str(value).lower()
text = text.replace("ё", "е")
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
return "SUV"
for marker, mapped in _BODY_MAP.items():
@@ -550,7 +529,7 @@ class MobileDeMapper:
parsed = cls._int_from_text(text)
if parsed and 500 <= parsed <= 9000:
return parsed
if re.search(r"(ccm|cm3|cm³|\bcc\b|куб|cubic|displacement)", text, re.IGNORECASE):
if re.search(r"(ccm|cm3|cm³|\bcc\b|cubic|displacement)", text, re.IGNORECASE):
parsed = cls._engine_volume_from_cc_text(text)
if parsed:
return parsed
@@ -573,7 +552,7 @@ class MobileDeMapper:
@classmethod
def _engine_volume_from_cc_text(cls, value: str) -> int | None:
text = str(value or "")
match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc|куб)", text, re.IGNORECASE)
match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc)", text, re.IGNORECASE)
if not match:
return None
parsed = cls._int_from_text(match.group(1))
@@ -597,7 +576,6 @@ class MobileDeMapper:
@staticmethod
def _normalize_color(value: Any) -> str:
text = "" if value is None else str(value).lower().strip()
text = text.replace("ё", "е")
text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
text = re.sub(r"[_\-/]+", " ", text)
for marker, mapped in _COLOR_MAP.items():
@@ -623,7 +601,7 @@ class MobileDeMapper:
@staticmethod
def _slugify(value: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
slug = re.sub(r"[^a-zA-Z0-9]+", "-", value.lower()).strip("-")
return slug[:180] or "mobilede-car"
@staticmethod