qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use Russian transport

Browse Source
This commit is contained in:
qananasikq
2026-08-18 13:10:47 +03:00
parent 8196fe50d2
commit e6c6732ed8
4 changed files with 223 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/client.py
+28 -10
View File
@@ -17,9 +17,11 @@ from .models import MobileDeListing, MobileDeSearchPage
logger = logging.getLogger("mobile_de.client")
BASE_URL = "https://suchen.mobile.de"
SEARCH_PATH = "/fahrzeuge/search.html"
DETAIL_PATH = "/fahrzeuge/details.html"
TRANSPORT_BASE_URL = "https://www.mobile.de"
TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
PUBLIC_BASE_URL = "https://suchen.mobile.de"
PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html"
DEFAULT_HEADERS = {
"user-agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -27,7 +29,7 @@ DEFAULT_HEADERS = {
"Chrome/124.0.0.0 Safari/537.36"
),
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US,en;q=0.9",
"accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7",
}
MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
@@ -103,11 +105,11 @@ class MobileDeClient:
"od": "up",
"vc": "Car",
"s": "Car",
"lang": "en",
"lang": "ru",
"pageNumber": page_number,
}
query.update({key: value for key, value in params.items() if value is not None})
return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}"
@staticmethod
def build_search_url_from_existing(
@@ -125,13 +127,26 @@ class MobileDeClient:
if page_number is not None:
query_items.append(("pageNumber", str(page_number)))
query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
query_items.append(("lang", "en"))
return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
query_items.append(("lang", "ru"))
return urlunsplit(
(
"https",
urlsplit(TRANSPORT_BASE_URL).netloc,
TRANSPORT_SEARCH_PATH,
urlencode(query_items),
"",
)
)
@staticmethod
def build_transport_detail_url(listing_id: str | int) -> str:
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"})
return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}"
@staticmethod
def build_detail_url(listing_id: str | int) -> str:
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
return f"{BASE_URL}{DETAIL_PATH}?{query}"
return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}"
def fetch_html(self, url: str, *, timeout: int = 30) -> str:
last_error: Exception | None = None
@@ -375,7 +390,10 @@ class MobileDeClient:
return ordered_pages
def fetch_detail(self, listing_id: str | int) -> dict:
html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
html = self.fetch_html(
self.build_transport_detail_url(listing_id),
timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
)
return extract_detail_listing(html)
@staticmethod
mobilede_scraper/mobile_de/mapper.py
+97 -14
View File
@@ -15,30 +15,42 @@ PARSER_ID_ALPHABET = ascii_letters + digits
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
_BODY_MAP = {
"кабриолет": "OPEN",
"родстер": "OPEN",
"cabrio": "OPEN",
"cabriolet": "OPEN",
"roadster": "OPEN",
"седан": "SEDAN",
"limousine": "SEDAN",
"saloon": "SEDAN",
"sedan": "SEDAN",
"внедорожник": "SUV",
"suv": "SUV",
"offroad": "SUV",
"gelandewagen": "SUV",
"geländewagen": "SUV",
"пикап": "PICKUP",
"pickup": "PICKUP",
"pick-up": "PICKUP",
"универсал": "STATION_WAGON",
"kombi": "STATION_WAGON",
"estatecar": "STATION_WAGON",
"touring": "STATION_WAGON",
"estate": "STATION_WAGON",
"минивэн": "MINIVAN",
"микроавтобус": "MINIVAN",
"фургон": "MINIVAN",
"van": "MINIVAN",
"kleinbus": "MINIVAN",
"bus": "MINIVAN",
"active tourer": "MINIVAN",
"gran tourer": "MINIVAN",
"купе": "COUPE",
"coupe": "COUPE",
"sportscar": "COUPE",
"sports car": "COUPE",
"хэтчбек": "HATCHBACK",
"малолитражный": "HATCHBACK",
"hatchback": "HATCHBACK",
"kleinwagen": "HATCHBACK",
"smallcar": "HATCHBACK",
@@ -48,13 +60,20 @@ _BODY_MAP = {
}
_GEARBOX_MAP = {
"автоматическая": "AT",
"автоматическая коробка": "AT",
"automatik": "AT",
"automatic": "AT",
"механическая": "MT",
"механическая коробка": "MT",
"manual": "MT",
"вариатор": "CVT",
"cvt": "CVT",
}
_COLOR_MAP = {
"черный": "black",
"чёрный": "black",
"schwarz": "black",
"saphirschwarz": "black",
"carbonschwarz": "black",
@@ -62,15 +81,19 @@ _COLOR_MAP = {
"jet black": "black",
"jetblack": "black",
"black": "black",
"белый": "white",
"weiss": "white",
"weiß": "white",
"alpinweiss": "white",
"alpine white": "white",
"mineralweiss": "white",
"white": "white",
"серебристый": "silver",
"серебряный": "silver",
"silber": "silver",
"argent": "silver",
"silver": "silver",
"серый": "gray",
"grau": "gray",
"grey": "gray",
"anthrazit": "gray",
@@ -83,31 +106,44 @@ _COLOR_MAP = {
"sophistograu": "gray",
"skyscraper grau": "gray",
"gray": "gray",
"красный": "red",
"rot": "red",
"burgundy": "red",
"bordeaux": "red",
"maroon": "red",
"red": "red",
"синий": "blue",
"голубой": "blue",
"blau": "blue",
"turquoise": "blue",
"cyan": "blue",
"blue": "blue",
"зеленый": "green",
"зелёный": "green",
"grün": "green",
"gruen": "green",
"green": "green",
"коричневый": "brown",
"braun": "brown",
"brown": "brown",
"бежевый": "beige",
"beige": "beige",
"champagner": "beige",
"champagne": "beige",
"creme": "beige",
"cream": "beige",
"ivory": "beige",
"желтый": "yellow",
"жёлтый": "yellow",
"gelb": "yellow",
"yellow": "yellow",
"оранжевый": "orange",
"orange": "orange",
"золотой": "gold",
"gold": "gold",
"бронзовый": "bronze",
"bronze": "bronze",
"фиолетовый": "purple",
"violett": "purple",
"lila": "purple",
"purple": "purple",
@@ -176,10 +212,7 @@ class MobileDeMapper:
or attr.get("damageCondition")
or attr.get("dc")
).lower()
is_damaged = (
bool(raw.get("hasDamage"))
or ("accident" in damage_text and "no accident" not in damage_text)
)
is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text)
year = self._year_from_first_registration(
self._first_present(
listing.first_registration,
@@ -248,7 +281,7 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=listing.url,
@@ -316,9 +349,11 @@ class MobileDeMapper:
)
damage_text = self._text(attrs.get("damageCondition")).lower()
is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text)
owners_text = self._text(attrs.get("numPreviousOwners"))
owners_text = self._text(
self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
)
one_owner = self._is_one_owner(owners_text)
return CarRecord(
@@ -349,7 +384,7 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=one_owner,
new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=MobileDeClient.build_detail_url(listing_id),
@@ -364,7 +399,7 @@ class MobileDeMapper:
details_confirmed=True,
images_confirmed=True,
preserve_existing_details=False,
images=self._images_from_listing(detail),
images=self._images_from_detail(detail),
)
@staticmethod
@@ -477,18 +512,18 @@ class MobileDeMapper:
@staticmethod
def _normalize_drive(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
return "FWD"
if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
return "RWD"
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
return "4WD"
return None
@staticmethod
def _normalize_country(value: Any) -> str:
text = "" if value is None else str(value).strip().upper()
if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}:
return "DE"
if text in {"US", "USA", "UNITED STATES"}:
return "US"
@@ -510,7 +545,10 @@ class MobileDeMapper:
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
return "SUV"
for marker, mapped in _BODY_MAP.items():
if marker in text:
if marker in {"van", "bus"}:
if re.search(rf"\b{re.escape(marker)}\b", text):
return mapped
elif marker in text:
return mapped
return "OTHER"
@@ -597,6 +635,41 @@ class MobileDeMapper:
text = "" if value is None else str(value).strip().lower()
return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
@staticmethod
def _bool_value(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return value != 0
text = "" if value is None else str(value).strip().lower()
if text in {"1", "true", "yes", "on", "да"}:
return True
if text in {"0", "false", "no", "off", "нет", ""}:
return False
return False
@classmethod
def _is_damaged_value(cls, flag: Any, description: Any) -> bool:
if cls._bool_value(flag):
return True
text = cls._text(description).lower()
if not text:
return False
if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")):
return False
return any(
marker in text
for marker in (
"дтп",
"аварийн",
"с повреждениями",
"устраненные повреждения",
"устранённые повреждения",
"accident",
"damaged",
)
)
@staticmethod
def _rating_text(value: Any) -> str | None:
if isinstance(value, dict):
@@ -658,6 +731,16 @@ class MobileDeMapper:
for index, source_url in enumerate(source_urls)
]
@staticmethod
def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]:
explicit_images = raw.get("images")
if isinstance(explicit_images, list) and explicit_images:
return MobileDeMapper._images_from_listing({"images": explicit_images})
media_gallery = raw.get("mediaGallery")
if isinstance(media_gallery, list) and media_gallery:
return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
return MobileDeMapper._images_from_listing(raw)
@staticmethod
def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
if depth > 8:
tests/test_mappers.py
+81
View File
@@ -434,6 +434,87 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.drive, "FWD")
def test_russian_search_values_map_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2001",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en",
title="Audi e-tron quattro",
raw={
"make": {"localized": "Audi"},
"model": {"localized": "e-tron"},
"price": {"grs": {"amount": 29700}},
"hasDamage": "false",
"isNew": "false",
"attr": {
"fr": "05/2021",
"ml": "59 000 км",
"ecol": "Черный Металлик",
"tr": "Автоматическая",
"c": "Внедорожник/Пикап",
},
},
)
)
self.assertEqual(record.brand, "Audi")
self.assertEqual(record.model, "e-tron")
self.assertEqual(record.year, 2021)
self.assertEqual(record.price, 29700)
self.assertEqual(record.mileage, 59000)
self.assertEqual(record.color, "black")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.body_type, "SUV")
self.assertFalse(record.is_damaged)
self.assertFalse(record.new_car)
def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"
record = self.mapper.detail_to_car_record(
"2002",
{
"shortTitle": "Porsche Macan",
"subTitle": "GTS",
"make": {"localized": "Porsche"},
"model": {"localized": "Macan"},
"price": {"grs": {"amount": 59900, "currency": "EUR"}},
"isConditionNew": "false",
"attributes": [
{"tag": "firstRegistration", "value": "04/2022"},
{"tag": "mileage", "value": "149 300 км"},
{"tag": "color", "value": "Синий Металлик"},
{"tag": "transmission", "value": "Автоматическая"},
{"tag": "category", "value": "Внедорожник/Пикап"},
{"tag": "cubicCapacity", "value": "2 894 ccm"},
{"tag": "numberOfPreviousOwners", "value": "1"},
{"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
],
"images": [{"uri": first}, {"uri": second}],
"links": {"preview": unrelated},
},
)
self.assertEqual(record.year, 2022)
self.assertEqual(record.mileage, 149300)
self.assertEqual(record.color, "blue")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 2894)
self.assertTrue(record.one_owner)
self.assertTrue(record.is_damaged)
self.assertFalse(record.new_car)
self.assertEqual(len(record.images), 2)
self.assertEqual(
[image.fullres_image for image in record.images],
[
f"https://{first}?rule=mo-1600",
f"https://{second}?rule=mo-1600",
],
)
if __name__ == "__main__":
unittest.main()
tests/test_mobilede_requirements.py
+17 -11
View File
@@ -47,20 +47,24 @@ class _Persistence:
class TestMobileDeRequirements(unittest.TestCase):
def test_english_urls_and_header_preserve_existing_query(self) -> None:
def test_russian_transport_and_english_public_url_preserve_query(self) -> None:
search = MobileDeClient.build_search_url_from_existing(
"https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep",
"https://suchen.mobile.de/fahrzeuge/search.html?ms=3500&lang=en&custom=keep",
page_number=3,
)
detail = MobileDeClient.build_detail_url("123")
self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9")
transport_detail = MobileDeClient.build_transport_detail_url("123")
public_detail = MobileDeClient.build_detail_url("123")
self.assertEqual(DEFAULT_HEADERS["accept-language"], "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7")
self.assertEqual(urlsplit(search).scheme, "https")
self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de")
self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html")
self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html")
self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"])
self.assertEqual(urlsplit(search).netloc, "www.mobile.de")
self.assertEqual(urlsplit(search).path, "/ru/транспортные-средства/поиск.html")
self.assertEqual(urlsplit(transport_detail).path, "/ru/транспортные-средства/подробности.html")
self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de")
self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html")
self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"])
self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"])
self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"])
self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"])
def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
origin_id = "mobile.de:123"
@@ -69,9 +73,11 @@ class TestMobileDeRequirements(unittest.TestCase):
self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
self.assertNotEqual(parser_id, origin_id)
def test_mapper_contains_no_cyrillic_literals(self) -> None:
def test_mapper_translates_russian_source_values_to_english_enums(self) -> None:
mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8")))
mapper_text = mapper_path.read_text(encoding="utf-8")
self.assertIn('"автоматическая": "AT"', mapper_text)
self.assertIn('"черный": "black"', mapper_text)
def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
with tempfile.TemporaryDirectory() as directory: