Use Russian transport

mobilede_scraper/mobile_de/client.py

@@ -17,9 +17,11 @@ from .models import MobileDeListing, MobileDeSearchPage
 
 logger = logging.getLogger("mobile_de.client")
 
-BASE_URL = "https://suchen.mobile.de"
+TRANSPORT_BASE_URL = "https://www.mobile.de"
-SEARCH_PATH = "/fahrzeuge/search.html"
+TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
-DETAIL_PATH = "/fahrzeuge/details.html"
+TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+PUBLIC_BASE_URL = "https://suchen.mobile.de"
+PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html"
 DEFAULT_HEADERS = {
     "user-agent": (
         "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -27,7 +29,7 @@ DEFAULT_HEADERS = {
         "Chrome/124.0.0.0 Safari/537.36"
     ),
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7",
 }
 MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
 MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
@@ -103,11 +105,11 @@ class MobileDeClient:
             "od": "up",
             "vc": "Car",
             "s": "Car",
-            "lang": "en",
+            "lang": "ru",
             "pageNumber": page_number,
         }
         query.update({key: value for key, value in params.items() if value is not None})
-        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+        return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}"
 
     @staticmethod
     def build_search_url_from_existing(
@@ -125,13 +127,26 @@ class MobileDeClient:
         if page_number is not None:
             query_items.append(("pageNumber", str(page_number)))
         query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
-        query_items.append(("lang", "en"))
+        query_items.append(("lang", "ru"))
-        return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
+        return urlunsplit(
+            (
+                "https",
+                urlsplit(TRANSPORT_BASE_URL).netloc,
+                TRANSPORT_SEARCH_PATH,
+                urlencode(query_items),
+                "",
+            )
+        )
+
+    @staticmethod
+    def build_transport_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"})
+        return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}"
 
     @staticmethod
     def build_detail_url(listing_id: str | int) -> str:
         query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
-        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+        return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}"
 
     def fetch_html(self, url: str, *, timeout: int = 30) -> str:
         last_error: Exception | None = None
@@ -375,7 +390,10 @@ class MobileDeClient:
         return ordered_pages
 
     def fetch_detail(self, listing_id: str | int) -> dict:
-        html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
+        html = self.fetch_html(
+            self.build_transport_detail_url(listing_id),
+            timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
+        )
         return extract_detail_listing(html)
 
     @staticmethod

mobilede_scraper/mobile_de/mapper.py

@@ -15,30 +15,42 @@ PARSER_ID_ALPHABET = ascii_letters + digits
 PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
 
 _BODY_MAP = {
+    "кабриолет": "OPEN",
+    "родстер": "OPEN",
     "cabrio": "OPEN",
     "cabriolet": "OPEN",
     "roadster": "OPEN",
+    "седан": "SEDAN",
     "limousine": "SEDAN",
     "saloon": "SEDAN",
     "sedan": "SEDAN",
+    "внедорожник": "SUV",
     "suv": "SUV",
     "offroad": "SUV",
     "gelandewagen": "SUV",
     "geländewagen": "SUV",
+    "пикап": "PICKUP",
     "pickup": "PICKUP",
     "pick-up": "PICKUP",
+    "универсал": "STATION_WAGON",
     "kombi": "STATION_WAGON",
     "estatecar": "STATION_WAGON",
     "touring": "STATION_WAGON",
     "estate": "STATION_WAGON",
+    "минивэн": "MINIVAN",
+    "микроавтобус": "MINIVAN",
+    "фургон": "MINIVAN",
     "van": "MINIVAN",
     "kleinbus": "MINIVAN",
     "bus": "MINIVAN",
     "active tourer": "MINIVAN",
     "gran tourer": "MINIVAN",
+    "купе": "COUPE",
     "coupe": "COUPE",
     "sportscar": "COUPE",
     "sports car": "COUPE",
+    "хэтчбек": "HATCHBACK",
+    "малолитражный": "HATCHBACK",
     "hatchback": "HATCHBACK",
     "kleinwagen": "HATCHBACK",
     "smallcar": "HATCHBACK",
@@ -48,13 +60,20 @@ _BODY_MAP = {
 }
 
 _GEARBOX_MAP = {
+    "автоматическая": "AT",
+    "автоматическая коробка": "AT",
     "automatik": "AT",
     "automatic": "AT",
+    "механическая": "MT",
+    "механическая коробка": "MT",
     "manual": "MT",
+    "вариатор": "CVT",
     "cvt": "CVT",
 }
 
 _COLOR_MAP = {
+    "черный": "black",
+    "чёрный": "black",
     "schwarz": "black",
     "saphirschwarz": "black",
     "carbonschwarz": "black",
@@ -62,15 +81,19 @@ _COLOR_MAP = {
     "jet black": "black",
     "jetblack": "black",
     "black": "black",
+    "белый": "white",
     "weiss": "white",
     "weiß": "white",
     "alpinweiss": "white",
     "alpine white": "white",
     "mineralweiss": "white",
     "white": "white",
+    "серебристый": "silver",
+    "серебряный": "silver",
     "silber": "silver",
     "argent": "silver",
     "silver": "silver",
+    "серый": "gray",
     "grau": "gray",
     "grey": "gray",
     "anthrazit": "gray",
@@ -83,31 +106,44 @@ _COLOR_MAP = {
     "sophistograu": "gray",
     "skyscraper grau": "gray",
     "gray": "gray",
+    "красный": "red",
     "rot": "red",
     "burgundy": "red",
     "bordeaux": "red",
     "maroon": "red",
     "red": "red",
+    "синий": "blue",
+    "голубой": "blue",
     "blau": "blue",
     "turquoise": "blue",
     "cyan": "blue",
     "blue": "blue",
+    "зеленый": "green",
+    "зелёный": "green",
     "grün": "green",
     "gruen": "green",
     "green": "green",
+    "коричневый": "brown",
     "braun": "brown",
     "brown": "brown",
+    "бежевый": "beige",
     "beige": "beige",
     "champagner": "beige",
     "champagne": "beige",
     "creme": "beige",
     "cream": "beige",
     "ivory": "beige",
+    "желтый": "yellow",
+    "жёлтый": "yellow",
     "gelb": "yellow",
     "yellow": "yellow",
+    "оранжевый": "orange",
     "orange": "orange",
+    "золотой": "gold",
     "gold": "gold",
+    "бронзовый": "bronze",
     "bronze": "bronze",
+    "фиолетовый": "purple",
     "violett": "purple",
     "lila": "purple",
     "purple": "purple",
@@ -176,10 +212,7 @@ class MobileDeMapper:
             or attr.get("damageCondition")
             or attr.get("dc")
         ).lower()
-        is_damaged = (
+        is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text)
-            bool(raw.get("hasDamage"))
-            or ("accident" in damage_text and "no accident" not in damage_text)
-        )
         year = self._year_from_first_registration(
             self._first_present(
                 listing.first_registration,
@@ -248,7 +281,7 @@ class MobileDeMapper:
             ),
             selling_type="STOCK",
             one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
-            new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
+            new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")),
             is_hidden=False,
             origin="MOBILEDE",
             origin_url=listing.url,
@@ -316,9 +349,11 @@ class MobileDeMapper:
         )
 
         damage_text = self._text(attrs.get("damageCondition")).lower()
-        is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
+        is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text)
 
-        owners_text = self._text(attrs.get("numPreviousOwners"))
+        owners_text = self._text(
+            self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
+        )
         one_owner = self._is_one_owner(owners_text)
 
         return CarRecord(
@@ -349,7 +384,7 @@ class MobileDeMapper:
             ),
             selling_type="STOCK",
             one_owner=one_owner,
-            new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
+            new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")),
             is_hidden=False,
             origin="MOBILEDE",
             origin_url=MobileDeClient.build_detail_url(listing_id),
@@ -364,7 +399,7 @@ class MobileDeMapper:
             details_confirmed=True,
             images_confirmed=True,
             preserve_existing_details=False,
-            images=self._images_from_listing(detail),
+            images=self._images_from_detail(detail),
         )
 
     @staticmethod
@@ -477,18 +512,18 @@ class MobileDeMapper:
     @staticmethod
     def _normalize_drive(value: Any) -> str | None:
         text = "" if value is None else str(value).lower()
-        if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
+        if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
             return "FWD"
-        if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
+        if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
             return "RWD"
-        if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
+        if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
             return "4WD"
         return None
 
     @staticmethod
     def _normalize_country(value: Any) -> str:
         text = "" if value is None else str(value).strip().upper()
-        if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
+        if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}:
             return "DE"
         if text in {"US", "USA", "UNITED STATES"}:
             return "US"
@@ -510,7 +545,10 @@ class MobileDeMapper:
         if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
             return "SUV"
         for marker, mapped in _BODY_MAP.items():
-            if marker in text:
+            if marker in {"van", "bus"}:
+                if re.search(rf"\b{re.escape(marker)}\b", text):
+                    return mapped
+            elif marker in text:
                 return mapped
         return "OTHER"
 
@@ -597,6 +635,41 @@ class MobileDeMapper:
         text = "" if value is None else str(value).strip().lower()
         return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
 
+    @staticmethod
+    def _bool_value(value: Any) -> bool:
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, (int, float)):
+            return value != 0
+        text = "" if value is None else str(value).strip().lower()
+        if text in {"1", "true", "yes", "on", "да"}:
+            return True
+        if text in {"0", "false", "no", "off", "нет", ""}:
+            return False
+        return False
+
+    @classmethod
+    def _is_damaged_value(cls, flag: Any, description: Any) -> bool:
+        if cls._bool_value(flag):
+            return True
+        text = cls._text(description).lower()
+        if not text:
+            return False
+        if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")):
+            return False
+        return any(
+            marker in text
+            for marker in (
+                "дтп",
+                "аварийн",
+                "с повреждениями",
+                "устраненные повреждения",
+                "устранённые повреждения",
+                "accident",
+                "damaged",
+            )
+        )
+
     @staticmethod
     def _rating_text(value: Any) -> str | None:
         if isinstance(value, dict):
@@ -658,6 +731,16 @@ class MobileDeMapper:
             for index, source_url in enumerate(source_urls)
         ]
 
+    @staticmethod
+    def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]:
+        explicit_images = raw.get("images")
+        if isinstance(explicit_images, list) and explicit_images:
+            return MobileDeMapper._images_from_listing({"images": explicit_images})
+        media_gallery = raw.get("mediaGallery")
+        if isinstance(media_gallery, list) and media_gallery:
+            return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
+        return MobileDeMapper._images_from_listing(raw)
+
     @staticmethod
     def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
         if depth > 8:

tests/test_mappers.py

@@ -434,6 +434,87 @@ class TestMobileDeMapper(unittest.TestCase):
 
         self.assertEqual(record.drive, "FWD")
 
+    def test_russian_search_values_map_to_english_contract(self) -> None:
+        record = self.mapper.listing_to_car_record(
+            MobileDeListing(
+                id="2001",
+                url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en",
+                title="Audi e-tron quattro",
+                raw={
+                    "make": {"localized": "Audi"},
+                    "model": {"localized": "e-tron"},
+                    "price": {"grs": {"amount": 29700}},
+                    "hasDamage": "false",
+                    "isNew": "false",
+                    "attr": {
+                        "fr": "05/2021",
+                        "ml": "59 000 км",
+                        "ecol": "Черный Металлик",
+                        "tr": "Автоматическая",
+                        "c": "Внедорожник/Пикап",
+                    },
+                },
+            )
+        )
+
+        self.assertEqual(record.brand, "Audi")
+        self.assertEqual(record.model, "e-tron")
+        self.assertEqual(record.year, 2021)
+        self.assertEqual(record.price, 29700)
+        self.assertEqual(record.mileage, 59000)
+        self.assertEqual(record.color, "black")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.drive, "4WD")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertFalse(record.is_damaged)
+        self.assertFalse(record.new_car)
+
+    def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
+        first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
+        second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
+        unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"
+        record = self.mapper.detail_to_car_record(
+            "2002",
+            {
+                "shortTitle": "Porsche Macan",
+                "subTitle": "GTS",
+                "make": {"localized": "Porsche"},
+                "model": {"localized": "Macan"},
+                "price": {"grs": {"amount": 59900, "currency": "EUR"}},
+                "isConditionNew": "false",
+                "attributes": [
+                    {"tag": "firstRegistration", "value": "04/2022"},
+                    {"tag": "mileage", "value": "149 300 км"},
+                    {"tag": "color", "value": "Синий Металлик"},
+                    {"tag": "transmission", "value": "Автоматическая"},
+                    {"tag": "category", "value": "Внедорожник/Пикап"},
+                    {"tag": "cubicCapacity", "value": "2 894 ccm"},
+                    {"tag": "numberOfPreviousOwners", "value": "1"},
+                    {"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
+                ],
+                "images": [{"uri": first}, {"uri": second}],
+                "links": {"preview": unrelated},
+            },
+        )
+
+        self.assertEqual(record.year, 2022)
+        self.assertEqual(record.mileage, 149300)
+        self.assertEqual(record.color, "blue")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.engine_volume, 2894)
+        self.assertTrue(record.one_owner)
+        self.assertTrue(record.is_damaged)
+        self.assertFalse(record.new_car)
+        self.assertEqual(len(record.images), 2)
+        self.assertEqual(
+            [image.fullres_image for image in record.images],
+            [
+                f"https://{first}?rule=mo-1600",
+                f"https://{second}?rule=mo-1600",
+            ],
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mobilede_requirements.py

@@ -47,20 +47,24 @@ class _Persistence:
 
 
 class TestMobileDeRequirements(unittest.TestCase):
-    def test_english_urls_and_header_preserve_existing_query(self) -> None:
+    def test_russian_transport_and_english_public_url_preserve_query(self) -> None:
         search = MobileDeClient.build_search_url_from_existing(
-            "https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep",
+            "https://suchen.mobile.de/fahrzeuge/search.html?ms=3500&lang=en&custom=keep",
             page_number=3,
         )
-        detail = MobileDeClient.build_detail_url("123")
+        transport_detail = MobileDeClient.build_transport_detail_url("123")
-        self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9")
+        public_detail = MobileDeClient.build_detail_url("123")
+        self.assertEqual(DEFAULT_HEADERS["accept-language"], "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7")
         self.assertEqual(urlsplit(search).scheme, "https")
-        self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(search).netloc, "www.mobile.de")
-        self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html")
+        self.assertEqual(urlsplit(search).path, "/ru/транспортные-средства/поиск.html")
-        self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(urlsplit(transport_detail).path, "/ru/транспортные-средства/подробности.html")
-        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"])
+        self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de")
+        self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html")
+        self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"])
         self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
-        self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"])
+        self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"])
+        self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"])
 
     def test_parser_id_is_deterministic_and_matches_project_format(self) -> None:
         origin_id = "mobile.de:123"
@@ -69,9 +73,11 @@ class TestMobileDeRequirements(unittest.TestCase):
         self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id))
         self.assertNotEqual(parser_id, origin_id)
 
-    def test_mapper_contains_no_cyrillic_literals(self) -> None:
+    def test_mapper_translates_russian_source_values_to_english_enums(self) -> None:
         mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py"
-        self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8")))
+        mapper_text = mapper_path.read_text(encoding="utf-8")
+        self.assertIn('"автоматическая": "AT"', mapper_text)
+        self.assertIn('"черный": "black"', mapper_text)
 
     def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None:
         with tempfile.TemporaryDirectory() as directory: