diff --git a/mobilede_scraper/mobile_de/client.py b/mobilede_scraper/mobile_de/client.py index 124028d..89c9ddb 100644 --- a/mobilede_scraper/mobile_de/client.py +++ b/mobilede_scraper/mobile_de/client.py @@ -17,9 +17,11 @@ from .models import MobileDeListing, MobileDeSearchPage logger = logging.getLogger("mobile_de.client") -BASE_URL = "https://suchen.mobile.de" -SEARCH_PATH = "/fahrzeuge/search.html" -DETAIL_PATH = "/fahrzeuge/details.html" +TRANSPORT_BASE_URL = "https://www.mobile.de" +TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html" +TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html" +PUBLIC_BASE_URL = "https://suchen.mobile.de" +PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html" DEFAULT_HEADERS = { "user-agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " @@ -27,7 +29,7 @@ DEFAULT_HEADERS = { "Chrome/124.0.0.0 Safari/537.36" ), "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "accept-language": "en-US,en;q=0.9", + "accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7", } MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4"))) MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2"))) @@ -103,11 +105,11 @@ class MobileDeClient: "od": "up", "vc": "Car", "s": "Car", - "lang": "en", + "lang": "ru", "pageNumber": page_number, } query.update({key: value for key, value in params.items() if value is not None}) - return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}" + return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}" @staticmethod def build_search_url_from_existing( @@ -125,13 +127,26 @@ class MobileDeClient: if page_number is not None: query_items.append(("pageNumber", str(page_number))) query_items.extend((key, str(value)) for key, value in params.items() if value is not None) - query_items.append(("lang", "en")) - return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), "")) + query_items.append(("lang", "ru")) + return urlunsplit( + ( + "https", + urlsplit(TRANSPORT_BASE_URL).netloc, + TRANSPORT_SEARCH_PATH, + urlencode(query_items), + "", + ) + ) + + @staticmethod + def build_transport_detail_url(listing_id: str | int) -> str: + query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"}) + return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}" @staticmethod def build_detail_url(listing_id: str | int) -> str: query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"}) - return f"{BASE_URL}{DETAIL_PATH}?{query}" + return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}" def fetch_html(self, url: str, *, timeout: int = 30) -> str: last_error: Exception | None = None @@ -375,7 +390,10 @@ class MobileDeClient: return ordered_pages def fetch_detail(self, listing_id: str | int) -> dict: - html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS) + html = self.fetch_html( + self.build_transport_detail_url(listing_id), + timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS, + ) return extract_detail_listing(html) @staticmethod diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py index 22682cf..b5939dc 100644 --- a/mobilede_scraper/mobile_de/mapper.py +++ b/mobilede_scraper/mobile_de/mapper.py @@ -15,30 +15,42 @@ PARSER_ID_ALPHABET = ascii_letters + digits PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$") _BODY_MAP = { + "кабриолет": "OPEN", + "родстер": "OPEN", "cabrio": "OPEN", "cabriolet": "OPEN", "roadster": "OPEN", + "седан": "SEDAN", "limousine": "SEDAN", "saloon": "SEDAN", "sedan": "SEDAN", + "внедорожник": "SUV", "suv": "SUV", "offroad": "SUV", "gelandewagen": "SUV", "geländewagen": "SUV", + "пикап": "PICKUP", "pickup": "PICKUP", "pick-up": "PICKUP", + "универсал": "STATION_WAGON", "kombi": "STATION_WAGON", "estatecar": "STATION_WAGON", "touring": "STATION_WAGON", "estate": "STATION_WAGON", + "минивэн": "MINIVAN", + "микроавтобус": "MINIVAN", + "фургон": "MINIVAN", "van": "MINIVAN", "kleinbus": "MINIVAN", "bus": "MINIVAN", "active tourer": "MINIVAN", "gran tourer": "MINIVAN", + "купе": "COUPE", "coupe": "COUPE", "sportscar": "COUPE", "sports car": "COUPE", + "хэтчбек": "HATCHBACK", + "малолитражный": "HATCHBACK", "hatchback": "HATCHBACK", "kleinwagen": "HATCHBACK", "smallcar": "HATCHBACK", @@ -48,13 +60,20 @@ _BODY_MAP = { } _GEARBOX_MAP = { + "автоматическая": "AT", + "автоматическая коробка": "AT", "automatik": "AT", "automatic": "AT", + "механическая": "MT", + "механическая коробка": "MT", "manual": "MT", + "вариатор": "CVT", "cvt": "CVT", } _COLOR_MAP = { + "черный": "black", + "чёрный": "black", "schwarz": "black", "saphirschwarz": "black", "carbonschwarz": "black", @@ -62,15 +81,19 @@ _COLOR_MAP = { "jet black": "black", "jetblack": "black", "black": "black", + "белый": "white", "weiss": "white", "weiß": "white", "alpinweiss": "white", "alpine white": "white", "mineralweiss": "white", "white": "white", + "серебристый": "silver", + "серебряный": "silver", "silber": "silver", "argent": "silver", "silver": "silver", + "серый": "gray", "grau": "gray", "grey": "gray", "anthrazit": "gray", @@ -83,31 +106,44 @@ _COLOR_MAP = { "sophistograu": "gray", "skyscraper grau": "gray", "gray": "gray", + "красный": "red", "rot": "red", "burgundy": "red", "bordeaux": "red", "maroon": "red", "red": "red", + "синий": "blue", + "голубой": "blue", "blau": "blue", "turquoise": "blue", "cyan": "blue", "blue": "blue", + "зеленый": "green", + "зелёный": "green", "grün": "green", "gruen": "green", "green": "green", + "коричневый": "brown", "braun": "brown", "brown": "brown", + "бежевый": "beige", "beige": "beige", "champagner": "beige", "champagne": "beige", "creme": "beige", "cream": "beige", "ivory": "beige", + "желтый": "yellow", + "жёлтый": "yellow", "gelb": "yellow", "yellow": "yellow", + "оранжевый": "orange", "orange": "orange", + "золотой": "gold", "gold": "gold", + "бронзовый": "bronze", "bronze": "bronze", + "фиолетовый": "purple", "violett": "purple", "lila": "purple", "purple": "purple", @@ -176,10 +212,7 @@ class MobileDeMapper: or attr.get("damageCondition") or attr.get("dc") ).lower() - is_damaged = ( - bool(raw.get("hasDamage")) - or ("accident" in damage_text and "no accident" not in damage_text) - ) + is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text) year = self._year_from_first_registration( self._first_present( listing.first_registration, @@ -248,7 +281,7 @@ class MobileDeMapper: ), selling_type="STOCK", one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")), - new_car=bool(raw.get("isNew") or raw.get("isConditionNew")), + new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")), is_hidden=False, origin="MOBILEDE", origin_url=listing.url, @@ -316,9 +349,11 @@ class MobileDeMapper: ) damage_text = self._text(attrs.get("damageCondition")).lower() - is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage")) + is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text) - owners_text = self._text(attrs.get("numPreviousOwners")) + owners_text = self._text( + self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners") + ) one_owner = self._is_one_owner(owners_text) return CarRecord( @@ -349,7 +384,7 @@ class MobileDeMapper: ), selling_type="STOCK", one_owner=one_owner, - new_car=bool(detail.get("isNew") or detail.get("isConditionNew")), + new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")), is_hidden=False, origin="MOBILEDE", origin_url=MobileDeClient.build_detail_url(listing_id), @@ -364,7 +399,7 @@ class MobileDeMapper: details_confirmed=True, images_confirmed=True, preserve_existing_details=False, - images=self._images_from_listing(detail), + images=self._images_from_detail(detail), ) @staticmethod @@ -477,18 +512,18 @@ class MobileDeMapper: @staticmethod def _normalize_drive(value: Any) -> str | None: text = "" if value is None else str(value).lower() - if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")): + if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")): return "FWD" - if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")): + if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")): return "RWD" - if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")): + if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")): return "4WD" return None @staticmethod def _normalize_country(value: Any) -> str: text = "" if value is None else str(value).strip().upper() - if text in {"DE", "GERMANY", "DEUTSCHLAND"}: + if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}: return "DE" if text in {"US", "USA", "UNITED STATES"}: return "US" @@ -510,7 +545,10 @@ class MobileDeMapper: if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text): return "SUV" for marker, mapped in _BODY_MAP.items(): - if marker in text: + if marker in {"van", "bus"}: + if re.search(rf"\b{re.escape(marker)}\b", text): + return mapped + elif marker in text: return mapped return "OTHER" @@ -597,6 +635,41 @@ class MobileDeMapper: text = "" if value is None else str(value).strip().lower() return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"} + @staticmethod + def _bool_value(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return value != 0 + text = "" if value is None else str(value).strip().lower() + if text in {"1", "true", "yes", "on", "да"}: + return True + if text in {"0", "false", "no", "off", "нет", ""}: + return False + return False + + @classmethod + def _is_damaged_value(cls, flag: Any, description: Any) -> bool: + if cls._bool_value(flag): + return True + text = cls._text(description).lower() + if not text: + return False + if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")): + return False + return any( + marker in text + for marker in ( + "дтп", + "аварийн", + "с повреждениями", + "устраненные повреждения", + "устранённые повреждения", + "accident", + "damaged", + ) + ) + @staticmethod def _rating_text(value: Any) -> str | None: if isinstance(value, dict): @@ -658,6 +731,16 @@ class MobileDeMapper: for index, source_url in enumerate(source_urls) ] + @staticmethod + def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]: + explicit_images = raw.get("images") + if isinstance(explicit_images, list) and explicit_images: + return MobileDeMapper._images_from_listing({"images": explicit_images}) + media_gallery = raw.get("mediaGallery") + if isinstance(media_gallery, list) and media_gallery: + return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery}) + return MobileDeMapper._images_from_listing(raw) + @staticmethod def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]: if depth > 8: diff --git a/tests/test_mappers.py b/tests/test_mappers.py index fe80f8b..243e019 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -434,6 +434,87 @@ class TestMobileDeMapper(unittest.TestCase): self.assertEqual(record.drive, "FWD") + def test_russian_search_values_map_to_english_contract(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="2001", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en", + title="Audi e-tron quattro", + raw={ + "make": {"localized": "Audi"}, + "model": {"localized": "e-tron"}, + "price": {"grs": {"amount": 29700}}, + "hasDamage": "false", + "isNew": "false", + "attr": { + "fr": "05/2021", + "ml": "59 000 км", + "ecol": "Черный Металлик", + "tr": "Автоматическая", + "c": "Внедорожник/Пикап", + }, + }, + ) + ) + + self.assertEqual(record.brand, "Audi") + self.assertEqual(record.model, "e-tron") + self.assertEqual(record.year, 2021) + self.assertEqual(record.price, 29700) + self.assertEqual(record.mileage, 59000) + self.assertEqual(record.color, "black") + self.assertEqual(record.gearbox, "AT") + self.assertEqual(record.drive, "4WD") + self.assertEqual(record.body_type, "SUV") + self.assertFalse(record.is_damaged) + self.assertFalse(record.new_car) + + def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None: + first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111" + second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222" + unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333" + record = self.mapper.detail_to_car_record( + "2002", + { + "shortTitle": "Porsche Macan", + "subTitle": "GTS", + "make": {"localized": "Porsche"}, + "model": {"localized": "Macan"}, + "price": {"grs": {"amount": 59900, "currency": "EUR"}}, + "isConditionNew": "false", + "attributes": [ + {"tag": "firstRegistration", "value": "04/2022"}, + {"tag": "mileage", "value": "149 300 км"}, + {"tag": "color", "value": "Синий Металлик"}, + {"tag": "transmission", "value": "Автоматическая"}, + {"tag": "category", "value": "Внедорожник/Пикап"}, + {"tag": "cubicCapacity", "value": "2 894 ccm"}, + {"tag": "numberOfPreviousOwners", "value": "1"}, + {"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"}, + ], + "images": [{"uri": first}, {"uri": second}], + "links": {"preview": unrelated}, + }, + ) + + self.assertEqual(record.year, 2022) + self.assertEqual(record.mileage, 149300) + self.assertEqual(record.color, "blue") + self.assertEqual(record.gearbox, "AT") + self.assertEqual(record.body_type, "SUV") + self.assertEqual(record.engine_volume, 2894) + self.assertTrue(record.one_owner) + self.assertTrue(record.is_damaged) + self.assertFalse(record.new_car) + self.assertEqual(len(record.images), 2) + self.assertEqual( + [image.fullres_image for image in record.images], + [ + f"https://{first}?rule=mo-1600", + f"https://{second}?rule=mo-1600", + ], + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mobilede_requirements.py b/tests/test_mobilede_requirements.py index be31668..6f4295f 100644 --- a/tests/test_mobilede_requirements.py +++ b/tests/test_mobilede_requirements.py @@ -47,20 +47,24 @@ class _Persistence: class TestMobileDeRequirements(unittest.TestCase): - def test_english_urls_and_header_preserve_existing_query(self) -> None: + def test_russian_transport_and_english_public_url_preserve_query(self) -> None: search = MobileDeClient.build_search_url_from_existing( - "https://www.mobile.de/ru/search.html?ms=3500&lang=de&custom=keep", + "https://suchen.mobile.de/fahrzeuge/search.html?ms=3500&lang=en&custom=keep", page_number=3, ) - detail = MobileDeClient.build_detail_url("123") - self.assertEqual(DEFAULT_HEADERS["accept-language"], "en-US,en;q=0.9") + transport_detail = MobileDeClient.build_transport_detail_url("123") + public_detail = MobileDeClient.build_detail_url("123") + self.assertEqual(DEFAULT_HEADERS["accept-language"], "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7") self.assertEqual(urlsplit(search).scheme, "https") - self.assertEqual(urlsplit(search).netloc, "suchen.mobile.de") - self.assertEqual(urlsplit(search).path, "/fahrzeuge/search.html") - self.assertEqual(urlsplit(detail).path, "/fahrzeuge/details.html") - self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["en"]) + self.assertEqual(urlsplit(search).netloc, "www.mobile.de") + self.assertEqual(urlsplit(search).path, "/ru/транспортные-средства/поиск.html") + self.assertEqual(urlsplit(transport_detail).path, "/ru/транспортные-средства/подробности.html") + self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de") + self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html") + self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"]) self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"]) - self.assertEqual(parse_qs(urlsplit(detail).query)["lang"], ["en"]) + self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"]) + self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"]) def test_parser_id_is_deterministic_and_matches_project_format(self) -> None: origin_id = "mobile.de:123" @@ -69,9 +73,11 @@ class TestMobileDeRequirements(unittest.TestCase): self.assertEqual(parser_id, MobileDeMapper._parser_id(origin_id)) self.assertNotEqual(parser_id, origin_id) - def test_mapper_contains_no_cyrillic_literals(self) -> None: + def test_mapper_translates_russian_source_values_to_english_enums(self) -> None: mapper_path = Path(__file__).parents[1] / "mobilede_scraper" / "mobile_de" / "mapper.py" - self.assertIsNone(re.search(r"[А-Яа-яЁё]", mapper_path.read_text(encoding="utf-8"))) + mapper_text = mapper_path.read_text(encoding="utf-8") + self.assertIn('"автоматическая": "AT"', mapper_text) + self.assertIn('"черный": "black"', mapper_text) def test_upsert_uses_origin_id_and_migrates_legacy_parser_id(self) -> None: with tempfile.TemporaryDirectory() as directory: