diff --git a/mobilede_scraper/mobile_de/flight.py b/mobilede_scraper/mobile_de/flight.py index 1f017ef..80def85 100644 --- a/mobilede_scraper/mobile_de/flight.py +++ b/mobilede_scraper/mobile_de/flight.py @@ -2,9 +2,104 @@ from __future__ import annotations import json import re +from html import unescape +from html.parser import HTMLParser from typing import Any +from urllib.parse import parse_qs, urlsplit NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL) +SEARCH_CARD_TEST_ID_RE = re.compile(r"^base-result-listing-\d+$") +VOID_HTML_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"} + + +class _SearchCardEnrichmentParser(HTMLParser): + """Collect title/subtitle from SSR cards while preserving the listing-ID relation.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.enrichment_by_id: dict[str, dict[str, str]] = {} + self._card: dict[str, Any] | None = None + self._card_depth = 0 + self._title_container_depth: int | None = None + self._title_text_depth = 0 + + @staticmethod + def _listing_id_from_href(href: str | None) -> str | None: + if not href: + return None + values = parse_qs(urlsplit(unescape(href)).query).get("id", []) + return str(values[0]).strip() if values and str(values[0]).strip() else None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attributes = dict(attrs) + test_id = attributes.get("data-testid") or "" + is_void = tag.lower() in VOID_HTML_TAGS + + if self._card is None: + if SEARCH_CARD_TEST_ID_RE.fullmatch(test_id): + self._card = {"id": None, "title": [], "subtitle": []} + self._card_depth = 0 if is_void else 1 + return + + if not is_void: + self._card_depth += 1 + + if tag.lower() == "a" and self._card.get("id") is None: + self._card["id"] = self._listing_id_from_href(attributes.get("href")) + + if test_id.endswith("-title"): + self._title_container_depth = self._card_depth + if test_id == "listing-title-card-view": + self._title_text_depth += 1 + + title_attribute = (attributes.get("title") or "").strip() + if title_attribute: + if test_id == "listing-title-card-view": + self._card["title"].append(title_attribute) + elif self._title_container_depth is not None: + self._card["subtitle"].append(title_attribute) + + def handle_data(self, data: str) -> None: + if self._card is not None and self._title_text_depth: + text = data.strip() + if text: + self._card["title"].append(text) + + def handle_endtag(self, tag: str) -> None: + if self._card is None or tag.lower() in VOID_HTML_TAGS: + return + if self._title_text_depth and tag.lower() == "span": + self._title_text_depth -= 1 + if self._title_container_depth == self._card_depth: + self._title_container_depth = None + self._card_depth -= 1 + if self._card_depth != 0: + return + + listing_id = self._card.get("id") + title = " ".join(dict.fromkeys(self._card["title"])).strip() + subtitle = " ".join(dict.fromkeys(self._card["subtitle"])).strip() + if listing_id and (title or subtitle): + self.enrichment_by_id[str(listing_id)] = { + "searchSsrTitle": title, + "searchSsrSubtitle": subtitle, + "searchSsrDriveText": " ".join(part for part in (title, subtitle) if part), + } + self._card = None + self._title_container_depth = None + self._title_text_depth = 0 + + +def extract_search_card_enrichment(html: str) -> dict[str, dict[str, str]]: + """Return SSR title/subtitle enrichment keyed by the Search listing ID. + + This reads only fields rendered inside the card's title block; it never uses + generic dealer, location, or arbitrary card text as a vehicle attribute. + """ + parser = _SearchCardEnrichmentParser() + parser.feed(html) + parser.close() + return parser.enrichment_by_id def extract_next_flight_strings(html: str) -> list[str]: @@ -77,4 +172,51 @@ def extract_detail_listing(html: str) -> dict[str, Any]: listing = extract_json_object_after(chunk, '"listing":') if isinstance(listing, dict): return listing + return extract_legacy_detail_listing(html) + + +def extract_legacy_detail_listing(html: str) -> dict[str, Any]: + """Extract and normalize the compact legacy VIP INITIAL_STATE payload.""" + state = extract_json_object_after(html, "window.__INITIAL_STATE__ =") + if not isinstance(state, dict): + return {} + + search = state.get("search") + vip = search.get("vip") if isinstance(search, dict) else None + ads = vip.get("ads") if isinstance(vip, dict) else None + if not isinstance(ads, dict): + return {} + + for payload in ads.values(): + data = payload.get("data") if isinstance(payload, dict) else None + ad = data.get("ad") if isinstance(data, dict) else None + if not isinstance(ad, dict): + continue + + listing = dict(ad) + make = listing.get("make") + if isinstance(make, str): + listing["make"] = {"localized": make} + model = listing.get("model") + if isinstance(model, str): + listing["model"] = {"localized": model} + + contact_info = listing.get("contactInfo") + if "contact" not in listing and isinstance(contact_info, dict): + listing["contact"] = contact_info + + gallery_images = listing.get("galleryImages") + if "images" not in listing and isinstance(gallery_images, list): + listing["images"] = gallery_images + + price = listing.get("price") + if isinstance(price, dict) and not isinstance(price.get("grs"), dict): + amount = price.get("grossAmount") + currency = price.get("grossCurrency") + if amount is not None or currency is not None: + normalized_price = dict(price) + normalized_price["grs"] = {"amount": amount, "currency": currency} + listing["price"] = normalized_price + + return listing return {} diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py index c382064..939e186 100644 --- a/mobilede_scraper/mobile_de/mapper.py +++ b/mobilede_scraper/mobile_de/mapper.py @@ -29,39 +29,6 @@ _RU_BODY_MAP = { "малолитражный": "HATCHBACK", } -_LEGACY_BODY_MAP = { - "cabrio": "OPEN", - "cabriolet": "OPEN", - "roadster": "OPEN", - "limousine": "SEDAN", - "saloon": "SEDAN", - "sedan": "SEDAN", - "suv": "SUV", - "offroad": "SUV", - "gelandewagen": "SUV", - "geländewagen": "SUV", - "pickup": "PICKUP", - "pick-up": "PICKUP", - "kombi": "STATION_WAGON", - "estatecar": "STATION_WAGON", - "touring": "STATION_WAGON", - "estate": "STATION_WAGON", - "van": "MINIVAN", - "kleinbus": "MINIVAN", - "bus": "MINIVAN", - "active tourer": "MINIVAN", - "gran tourer": "MINIVAN", - "coupe": "COUPE", - "sportscar": "COUPE", - "sports car": "COUPE", - "hatchback": "HATCHBACK", - "kleinwagen": "HATCHBACK", - "smallcar": "HATCHBACK", - "small car": "HATCHBACK", - "compact": "HATCHBACK", - "compactcar": "HATCHBACK", -} - _RU_GEARBOX_MAP = { "автоматическая": "AT", "автоматическая коробка": "AT", @@ -71,13 +38,6 @@ _RU_GEARBOX_MAP = { "вариатор": "CVT", } -_LEGACY_GEARBOX_MAP = { - "automatik": "AT", - "automatic": "AT", - "manual": "MT", - "cvt": "CVT", -} - _RU_COLOR_MAP = { "черный": "black", "чёрный": "black", @@ -101,65 +61,6 @@ _RU_COLOR_MAP = { "фиолетовый": "purple", } -_LEGACY_COLOR_MAP = { - "schwarz": "black", - "saphirschwarz": "black", - "carbonschwarz": "black", - "obsidianschwarz": "black", - "jet black": "black", - "jetblack": "black", - "black": "black", - "weiss": "white", - "weiß": "white", - "alpinweiss": "white", - "alpine white": "white", - "mineralweiss": "white", - "white": "white", - "silber": "silver", - "argent": "silver", - "silver": "silver", - "grau": "gray", - "grey": "gray", - "anthrazit": "gray", - "anthracite": "gray", - "graphit": "gray", - "graphite": "gray", - "spacegrau": "gray", - "brooklyn grau": "gray", - "brooklyn grey": "gray", - "sophistograu": "gray", - "skyscraper grau": "gray", - "gray": "gray", - "rot": "red", - "burgundy": "red", - "bordeaux": "red", - "maroon": "red", - "red": "red", - "blau": "blue", - "turquoise": "blue", - "cyan": "blue", - "blue": "blue", - "grün": "green", - "gruen": "green", - "green": "green", - "braun": "brown", - "brown": "brown", - "beige": "beige", - "champagner": "beige", - "champagne": "beige", - "creme": "beige", - "cream": "beige", - "ivory": "beige", - "gelb": "yellow", - "yellow": "yellow", - "orange": "orange", - "gold": "gold", - "bronze": "bronze", - "violett": "purple", - "lila": "purple", - "purple": "purple", -} - _RU_DRIVE_MARKERS = { "FWD": ("передний привод", "привод на передние колеса", "передние колеса"), "RWD": ("задний привод", "привод на задние колеса", "задние колеса"), @@ -167,12 +68,12 @@ _RU_DRIVE_MARKERS = { "2WD": ("привод на два колеса", "два ведущих колеса"), } -_LEGACY_DRIVE_MARKERS = { - "FWD": ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel"), - "RWD": ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel"), - "4WD": ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel"), - "2WD": ("sdrive", "2wd", "two-wheel drive", "two wheel drive", "zweiradantrieb"), -} +_SEARCH_SSR_DRIVE_MARKERS = ( + ("4WD", re.compile(r"\b(?:xdrive(?:\b|\d)|quattro|4matic|4motion|all4|awd|4wd|4x4|allrad(?:antrieb)?|carrera\s*4s?)\b", re.IGNORECASE)), + ("FWD", re.compile(r"\bfrontantrieb\b", re.IGNORECASE)), + ("RWD", re.compile(r"\bheckantrieb\b", re.IGNORECASE)), + ("2WD", re.compile(r"\bsdrive(?:\b|\d)", re.IGNORECASE)), +) _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"] _MOBILEDE_IMAGE_HOST = "img.classistatic.de" @@ -226,21 +127,16 @@ class MobileDeMapper: drive_text = " ".join( part for part in [ - listing.title, - listing.subtitle, - self._mapping_value(attr, "an"), attr_drive, raw.get("wheelDrive"), raw.get("drivetrain"), raw.get("drive"), raw.get("driveType"), - raw.get("modelDescription"), - raw.get("variant"), - raw.get("trim"), nested_drive, ] if isinstance(part, str) and part.strip() ) + search_ssr_drive = self._normalize_search_ssr_drive(raw.get("searchSsrDriveText")) damage_text = self._text( raw.get("damageCondition") or attr.get("damageCondition") @@ -289,7 +185,7 @@ class MobileDeMapper: nested_color, ) ), - drive=self._normalize_drive(drive_text), + drive=self._normalize_drive(drive_text) or search_ssr_drive, gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox), steering_wheel=self._normalize_steering( steering_text, @@ -381,11 +277,10 @@ class MobileDeMapper: detail.get("drive"), detail.get("driveType"), self._find_first_value(detail, _DRIVE_VALUE_KEYS), - feature_text, ] if isinstance(part, str) and part.strip() ) - drive = self._normalize_drive(explicit_drive_text) or self._normalize_drive(f"{short_title} {subtitle}") + drive = self._normalize_drive(explicit_drive_text) damage_text = self._text(attrs.get("damageCondition")).lower() is_damaged = self._is_damaged_value( @@ -455,14 +350,12 @@ class MobileDeMapper: "прокатный автомобиль", "арендный автомобиль", "rental vehicle", - "mietwagen", ), repair_history=self._contains_marker( damage_text, "устраненные повреждения", "устранённые повреждения", "repaired damage", - "reparierter unfallschaden", ), slug=self._slugify(" ".join(part for part in [title or f"{brand} {model}", str(year) if year is not None else ""] if part)), last_seen_at=datetime.now(timezone.utc), @@ -574,36 +467,43 @@ class MobileDeMapper: @staticmethod def _normalize_gearbox(value: Any) -> str | None: text = "" if value is None else str(value).lower() - for value_map in (_RU_GEARBOX_MAP, _LEGACY_GEARBOX_MAP): - for marker, mapped in value_map.items(): - if marker in text: - return mapped + for marker, mapped in _RU_GEARBOX_MAP.items(): + if marker in text: + return mapped return None @staticmethod def _normalize_drive(value: Any) -> str | None: text = "" if value is None else str(value).lower() - for marker_map in (_RU_DRIVE_MARKERS, _LEGACY_DRIVE_MARKERS): - for mapped, markers in marker_map.items(): - if any(marker in text for marker in markers): - return mapped + for mapped, markers in _RU_DRIVE_MARKERS.items(): + if any(marker in text for marker in markers): + return mapped + return None + + @staticmethod + def _normalize_search_ssr_drive(value: Any) -> str | None: + """Map only explicit, high-confidence drive tokens rendered by Search SSR.""" + text = "" if value is None else str(value) + for mapped, pattern in _SEARCH_SSR_DRIVE_MARKERS: + if pattern.search(text): + return mapped return None @classmethod - def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str: - if right_hand_drive is not None: - return "RIGHT" if cls._bool_value(right_hand_drive) else "LEFT" - text = "" if value is None else str(value).lower() - if any(marker in text for marker in ("правый руль", "праворуль", "right-hand drive", "right hand drive", "rhd", "rechtslenker", "lenkrad rechts")): + def _normalize_steering(cls, value: Any, *, right_hand_drive: Any = None) -> str | None: + if right_hand_drive is not None and cls._bool_value(right_hand_drive): return "RIGHT" - if any(marker in text for marker in ("левый руль", "леворуль", "left-hand drive", "left hand drive", "lhd", "linkslenker", "lenkrad links")): + text = "" if value is None else str(value).lower() + if any(marker in text for marker in ("правый руль", "праворуль")): + return "RIGHT" + if any(marker in text for marker in ("левый руль", "леворуль")): return "LEFT" - return "LEFT" + return None @staticmethod def _normalize_country(value: Any) -> str: text = "" if value is None else str(value).strip().upper() - if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}: + if text in {"DE", "ГЕРМАНИЯ"}: return "DE" if text in {"US", "USA", "UNITED STATES"}: return "US" @@ -624,13 +524,9 @@ class MobileDeMapper: text = "" if value is None else str(value).lower() if re.search(r"\bbmw\s+(?:x(?:[1-7]|m)|ix(?:[1-3])?|xm)\b", text): return "SUV" - for value_map in (_RU_BODY_MAP, _LEGACY_BODY_MAP): - for marker, mapped in value_map.items(): - if marker in {"van", "bus"}: - if re.search(rf"\b{re.escape(marker)}\b", text): - return mapped - elif marker in text: - return mapped + for marker, mapped in _RU_BODY_MAP.items(): + if marker in text: + return mapped return "OTHER" @staticmethod @@ -709,14 +605,10 @@ class MobileDeMapper: @staticmethod def _normalize_color(value: Any) -> str: text = "" if value is None else str(value).lower().strip() - text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss") text = re.sub(r"[_\-/]+", " ", text) - if text == "wei": - return "white" - for value_map in (_RU_COLOR_MAP, _LEGACY_COLOR_MAP): - for marker, mapped in value_map.items(): - if marker in text: - return mapped + for marker, mapped in _RU_COLOR_MAP.items(): + if marker in text: + return mapped return "other" @staticmethod @@ -830,6 +722,10 @@ class MobileDeMapper: return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery}) return MobileDeMapper._images_from_listing(raw) + def detail_to_images(self, raw: dict[str, Any]) -> list[ImageRecord]: + """Extract the complete gallery without mapping Detail car fields.""" + return self._images_from_detail(raw) + @staticmethod def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]: if depth > 8: diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 5e2e30f..f70b33a 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -150,13 +150,13 @@ class TestMobileDeMapper(unittest.TestCase): self.assertIsNone(record.drive) self.assertIsNone(record.gearbox) - # Нормализация регистра и пробелов. + # Нормализация регистра и пробелов для русского значения. record2 = self.mapper.listing_to_car_record( MobileDeListing( id="888", url="https://suchen.mobile.de/fahrzeuge/details.html?id=888", title="Honda Civic", - transmission=" Automatik ", + transmission=" Автоматическая ", ) ) self.assertEqual(record2.gearbox, "AT") @@ -188,9 +188,10 @@ class TestMobileDeMapper(unittest.TestCase): "attr": { "yc": "2016", "cn": "DE", - "ecol": "Silber", - "c": "OffRoad", + "ecol": "Серебристый", + "c": "Внедорожник", "cc": "1 998 ccm", + "wheelDrive": "Полный привод", } }, ) @@ -210,7 +211,7 @@ class TestMobileDeMapper(unittest.TestCase): url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002", title="BMW 320 Touring", subtitle="EstateCar", - raw={"attr": {"ecol": "Braun"}}, + raw={"attr": {"ecol": "Коричневый", "c": "Универсал"}}, ) ) @@ -230,12 +231,13 @@ class TestMobileDeMapper(unittest.TestCase): "hasDamage": True, "isConditionNew": True, "cubicCapacity": "1 995 ccm", - "color": "Black Metallic", + "color": "Черный Металлик", "attr": { "yc": "2020", "ml": "55 000 km", - "c": "SmallCar", - "tr": "Manual", + "c": "Малолитражный", + "tr": "Механическая", + "wheelDrive": "Передний привод", "pvo": "1", }, }, @@ -258,8 +260,8 @@ class TestMobileDeMapper(unittest.TestCase): self.assertFalse(record.non_smoking) self.assertEqual(record.evaluation, "GOOD_PRICE") - def test_drive_mapping_uses_specific_markers_only(self) -> None: - allrad = self.mapper.listing_to_car_record( + def test_drive_mapping_does_not_use_free_text(self) -> None: + title_drive = self.mapper.listing_to_car_record( MobileDeListing( id="1004", url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004", @@ -274,7 +276,62 @@ class TestMobileDeMapper(unittest.TestCase): ) ) - self.assertEqual(allrad.drive, "4WD") + self.assertIsNone(title_drive.drive) + self.assertIsNone(unrelated.drive) + + def test_fwd_and_rwd_are_not_inferred_from_free_text(self) -> None: + front = self.mapper.listing_to_car_record( + MobileDeListing( + id="10041", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10041", + title="BMW 120 Передний привод", + ) + ) + rear = self.mapper.listing_to_car_record( + MobileDeListing( + id="10042", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10042", + title="BMW 320", + subtitle="Задний привод", + ) + ) + + self.assertIsNone(front.drive) + self.assertIsNone(rear.drive) + + def test_search_ssr_drive_mapping_uses_only_explicit_high_confidence_tokens(self) -> None: + xdrive = self.mapper.listing_to_car_record( + MobileDeListing( + id="10043", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10043", + raw={"searchSsrDriveText": "BMW M135 i xDrive"}, + ) + ) + sdrive = self.mapper.listing_to_car_record( + MobileDeListing( + id="10044", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10044", + raw={"searchSsrDriveText": "BMW Z4 sDrive30i"}, + ) + ) + front = self.mapper.listing_to_car_record( + MobileDeListing( + id="10045", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10045", + raw={"searchSsrDriveText": "Kompaktwagen Frontantrieb"}, + ) + ) + unrelated = self.mapper.listing_to_car_record( + MobileDeListing( + id="10046", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=10046", + raw={"searchSsrDriveText": "BMW 320 All inclusive"}, + ) + ) + + self.assertEqual(xdrive.drive, "4WD") + self.assertEqual(sdrive.drive, "2WD") + self.assertEqual(front.drive, "FWD") self.assertIsNone(unrelated.drive) def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None: @@ -306,7 +363,7 @@ class TestMobileDeMapper(unittest.TestCase): url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007", title="Honda Civic", subtitle="1.6 Automatik", - raw={"attr": {"cn": "AT", "c": "SmallCar"}}, + raw={"attr": {"cn": "AT", "c": "Малолитражный"}}, ) ) @@ -349,11 +406,11 @@ class TestMobileDeMapper(unittest.TestCase): raw={ "vehicle": { "specs": { - "exteriorColor": "Silver", - "bodyType": "Cabrio", + "exteriorColor": "Серебристый", + "bodyType": "Кабриолет", "cubicCapacity": "1998", - "wheelDrive": "Allrad", - "transmission": "Automatic", + "wheelDrive": "Полный привод", + "transmission": "Автоматическая", } } }, @@ -375,8 +432,8 @@ class TestMobileDeMapper(unittest.TestCase): subtitle="2.0 l xDrive", raw={ "attr": { - "exteriorColor": "Skyscraper Grey Metallic", - "wheelDrive": "Antrieb vorne", + "exteriorColor": "Серый Металлик", + "wheelDrive": "Передний привод", "engineDisplacement": "2.0 l", }, "driveType": "xDrive", @@ -399,12 +456,12 @@ class TestMobileDeMapper(unittest.TestCase): "price": {"grs": {"amount": "31990", "currency": "EUR"}}, "contact": {"countryCode": "DE"}, "attributes": [ - {"tag": "exteriorColor", "value": "Brooklyn Grey"}, - {"tag": "driveType", "value": "All wheel drive"}, + {"tag": "exteriorColor", "value": "Серый"}, + {"tag": "driveType", "value": "Полный привод"}, {"tag": "engineDisplacement", "value": "2993 cc"}, {"tag": "mileage", "value": "145 000 km"}, {"tag": "year", "value": "2019"}, - {"tag": "bodyType", "value": "Geländewagen"}, + {"tag": "bodyType", "value": "Внедорожник"}, ], }, ) @@ -428,13 +485,13 @@ class TestMobileDeMapper(unittest.TestCase): id="1013", url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013", title="BMW 320 xDrive", - raw={"attr": {"wheelDrive": "Front-wheel drive"}}, + raw={"attr": {"wheelDrive": "Передний привод"}}, ) ) self.assertEqual(record.drive, "FWD") - def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None: + def test_bmw_sdrive_in_title_does_not_define_drive(self) -> None: record = self.mapper.detail_to_car_record( "1014", { @@ -443,7 +500,7 @@ class TestMobileDeMapper(unittest.TestCase): }, ) - self.assertEqual(record.drive, "2WD") + self.assertIsNone(record.drive) def test_russian_search_values_map_to_english_contract(self) -> None: record = self.mapper.listing_to_car_record( @@ -463,6 +520,7 @@ class TestMobileDeMapper(unittest.TestCase): "ecol": "Черный Металлик", "tr": "Автоматическая", "c": "Внедорожник/Пикап", + "wheelDrive": "Полный привод", }, }, ) @@ -492,7 +550,7 @@ class TestMobileDeMapper(unittest.TestCase): self.assertEqual(record.color, "gold") - def test_truncated_white_color_maps_to_english_contract(self) -> None: + def test_german_color_is_not_mapped(self) -> None: record = self.mapper.listing_to_car_record( MobileDeListing( id="2003", @@ -502,7 +560,7 @@ class TestMobileDeMapper(unittest.TestCase): ) ) - self.assertEqual(record.color, "white") + self.assertEqual(record.color, "other") def test_unknown_color_maps_to_other(self) -> None: record = self.mapper.listing_to_car_record( @@ -538,6 +596,44 @@ class TestMobileDeMapper(unittest.TestCase): self.assertEqual(record.steering_wheel, "RIGHT") + def test_false_right_hand_drive_does_not_invent_left(self) -> None: + record = self.mapper.detail_to_car_record( + "2007", + { + "shortTitle": "BMW 320", + "rightHandDrive": False, + }, + ) + + self.assertIsNone(record.steering_wheel) + + def test_missing_steering_does_not_invent_left(self) -> None: + record = self.mapper.detail_to_car_record("2008", {"shortTitle": "BMW 320"}) + + self.assertIsNone(record.steering_wheel) + + def test_english_and_german_enum_values_are_not_mapped(self) -> None: + record = self.mapper.listing_to_car_record( + MobileDeListing( + id="2009", + url="https://suchen.mobile.de/fahrzeuge/details.html?id=2009", + title="BMW 320 xDrive", + raw={ + "attr": { + "ecol": "Schwarz", + "tr": "Automatik", + "c": "Kombi", + "wheelDrive": "Allrad", + } + }, + ) + ) + + self.assertEqual(record.color, "other") + self.assertIsNone(record.gearbox) + self.assertEqual(record.body_type, "OTHER") + self.assertIsNone(record.drive) + def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None: first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111" second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222" @@ -565,6 +661,7 @@ class TestMobileDeMapper(unittest.TestCase): {"tag": "cubicCapacity", "value": "2 894 ccm"}, {"tag": "numberOfPreviousOwners", "value": "1"}, {"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"}, + {"tag": "wheelDrive", "value": "Привод на четыре колеса"}, ], "images": [{"uri": first}, {"uri": second}], "links": {"preview": unrelated}, diff --git a/tests/test_mobilede_requirements.py b/tests/test_mobilede_requirements.py index 6f4295f..bb1f814 100644 --- a/tests/test_mobilede_requirements.py +++ b/tests/test_mobilede_requirements.py @@ -49,7 +49,9 @@ class _Persistence: class TestMobileDeRequirements(unittest.TestCase): def test_russian_transport_and_english_public_url_preserve_query(self) -> None: search = MobileDeClient.build_search_url_from_existing( - "https://suchen.mobile.de/fahrzeuge/search.html?ms=3500&lang=en&custom=keep", + "https://www.mobile.de/ru/транспортные-средства/поиск.html" + "?isSearchRequest=true&s=Car&vc=Car" + "&ms=25100&ms=8600&ms=15200&ms=3500&ref=dsp&lang=en&custom=keep", page_number=3, ) transport_detail = MobileDeClient.build_transport_detail_url("123") @@ -62,6 +64,9 @@ class TestMobileDeRequirements(unittest.TestCase): self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de") self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html") self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"]) + self.assertEqual(parse_qs(urlsplit(search).query)["pageNumber"], ["3"]) + self.assertEqual(parse_qs(urlsplit(search).query)["ms"], ["25100", "8600", "15200", "3500"]) + self.assertEqual(parse_qs(urlsplit(search).query)["ref"], ["dsp"]) self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"]) self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"]) self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"]) diff --git a/tests/test_mobilede_scraper.py b/tests/test_mobilede_scraper.py index f4e433c..888a9e3 100644 --- a/tests/test_mobilede_scraper.py +++ b/tests/test_mobilede_scraper.py @@ -4,6 +4,7 @@ import unittest from datetime import datetime, timezone from unittest.mock import patch +from mobilede_scraper.mobile_de.client import MobileDeClient from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage from mobilede_scraper.mobile_de.scraper import MobileDeScraper @@ -61,9 +62,9 @@ class _FakeClient: "attributes": [ {"tag": "firstRegistration", "value": "05/2016"}, {"tag": "mileage", "value": "71 500 km"}, - {"tag": "category", "value": "OffRoad"}, + {"tag": "category", "value": "Внедорожник"}, {"tag": "color", "value": "Серый"}, - {"tag": "wheelDrive", "value": "xDrive"}, + {"tag": "wheelDrive", "value": "Полный привод"}, {"tag": "cubicCapacity", "value": "1 998 ccm"}, ], "images": ["https://img.example.test/1.jpg"], @@ -71,6 +72,15 @@ class _FakeClient: class TestMobileDeScraperStreamingSync(unittest.TestCase): + def test_worker_client_uses_configured_bounded_pool(self) -> None: + with patch.dict("os.environ", {"MOBILEDE_HTTP_POOL_SIZE": "7"}): + client = MobileDeClient.for_worker() + + adapter = client.session.get_adapter("https://") + self.assertEqual(adapter._pool_connections, 7) + self.assertEqual(adapter._pool_maxsize, 7) + self.assertTrue(adapter._pool_block) + def test_sync_search_upserts_each_page_during_run(self) -> None: pages = [ MobileDeSearchPage( @@ -151,7 +161,7 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase): id="2", url="https://example.test/2", title="BMW 320", - raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}}, + raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "Универсал", "ecol": "Серый", "cc": "1 998 ccm"}}, ), ], ),