qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enrich search cards

Browse Source
This commit is contained in:
qananasikq
2026-08-19 13:28:00 +03:00
parent 2913199d1e
commit 1b51618d94
5 changed files with 326 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_mappers.py
+123 -26
View File
@@ -150,13 +150,13 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertIsNone(record.drive)
self.assertIsNone(record.gearbox)
# Нормализация регистра и пробелов.
# Нормализация регистра и пробелов для русского значения.
record2 = self.mapper.listing_to_car_record(
MobileDeListing(
id="888",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
title="Honda Civic",
transmission=" Automatik ",
transmission=" Автоматическая ",
)
)
self.assertEqual(record2.gearbox, "AT")
@@ -188,9 +188,10 @@ class TestMobileDeMapper(unittest.TestCase):
"attr": {
"yc": "2016",
"cn": "DE",
"ecol": "Silber",
"c": "OffRoad",
"ecol": "Серебристый",
"c": "Внедорожник",
"cc": "1 998 ccm",
"wheelDrive": "Полный привод",
}
},
)
@@ -210,7 +211,7 @@ class TestMobileDeMapper(unittest.TestCase):
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
title="BMW 320 Touring",
subtitle="EstateCar",
raw={"attr": {"ecol": "Braun"}},
raw={"attr": {"ecol": "Коричневый", "c": "Универсал"}},
)
)
@@ -230,12 +231,13 @@ class TestMobileDeMapper(unittest.TestCase):
"hasDamage": True,
"isConditionNew": True,
"cubicCapacity": "1 995 ccm",
"color": "Black Metallic",
"color": "Черный Металлик",
"attr": {
"yc": "2020",
"ml": "55 000 km",
"c": "SmallCar",
"tr": "Manual",
"c": "Малолитражный",
"tr": "Механическая",
"wheelDrive": "Передний привод",
"pvo": "1",
},
},
@@ -258,8 +260,8 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertFalse(record.non_smoking)
self.assertEqual(record.evaluation, "GOOD_PRICE")
def test_drive_mapping_uses_specific_markers_only(self) -> None:
allrad = self.mapper.listing_to_car_record(
def test_drive_mapping_does_not_use_free_text(self) -> None:
title_drive = self.mapper.listing_to_car_record(
MobileDeListing(
id="1004",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
@@ -274,7 +276,62 @@ class TestMobileDeMapper(unittest.TestCase):
)
)
self.assertEqual(allrad.drive, "4WD")
self.assertIsNone(title_drive.drive)
self.assertIsNone(unrelated.drive)
def test_fwd_and_rwd_are_not_inferred_from_free_text(self) -> None:
front = self.mapper.listing_to_car_record(
MobileDeListing(
id="10041",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10041",
title="BMW 120 Передний привод",
)
)
rear = self.mapper.listing_to_car_record(
MobileDeListing(
id="10042",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10042",
title="BMW 320",
subtitle="Задний привод",
)
)
self.assertIsNone(front.drive)
self.assertIsNone(rear.drive)
def test_search_ssr_drive_mapping_uses_only_explicit_high_confidence_tokens(self) -> None:
xdrive = self.mapper.listing_to_car_record(
MobileDeListing(
id="10043",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10043",
raw={"searchSsrDriveText": "BMW M135 i xDrive"},
)
)
sdrive = self.mapper.listing_to_car_record(
MobileDeListing(
id="10044",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10044",
raw={"searchSsrDriveText": "BMW Z4 sDrive30i"},
)
)
front = self.mapper.listing_to_car_record(
MobileDeListing(
id="10045",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10045",
raw={"searchSsrDriveText": "Kompaktwagen Frontantrieb"},
)
)
unrelated = self.mapper.listing_to_car_record(
MobileDeListing(
id="10046",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=10046",
raw={"searchSsrDriveText": "BMW 320 All inclusive"},
)
)
self.assertEqual(xdrive.drive, "4WD")
self.assertEqual(sdrive.drive, "2WD")
self.assertEqual(front.drive, "FWD")
self.assertIsNone(unrelated.drive)
def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
@@ -306,7 +363,7 @@ class TestMobileDeMapper(unittest.TestCase):
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
title="Honda Civic",
subtitle="1.6 Automatik",
raw={"attr": {"cn": "AT", "c": "SmallCar"}},
raw={"attr": {"cn": "AT", "c": "Малолитражный"}},
)
)
@@ -349,11 +406,11 @@ class TestMobileDeMapper(unittest.TestCase):
raw={
"vehicle": {
"specs": {
"exteriorColor": "Silver",
"bodyType": "Cabrio",
"exteriorColor": "Серебристый",
"bodyType": "Кабриолет",
"cubicCapacity": "1998",
"wheelDrive": "Allrad",
"transmission": "Automatic",
"wheelDrive": "Полный привод",
"transmission": "Автоматическая",
}
}
},
@@ -375,8 +432,8 @@ class TestMobileDeMapper(unittest.TestCase):
subtitle="2.0 l xDrive",
raw={
"attr": {
"exteriorColor": "Skyscraper Grey Metallic",
"wheelDrive": "Antrieb vorne",
"exteriorColor": "Серый Металлик",
"wheelDrive": "Передний привод",
"engineDisplacement": "2.0 l",
},
"driveType": "xDrive",
@@ -399,12 +456,12 @@ class TestMobileDeMapper(unittest.TestCase):
"price": {"grs": {"amount": "31990", "currency": "EUR"}},
"contact": {"countryCode": "DE"},
"attributes": [
{"tag": "exteriorColor", "value": "Brooklyn Grey"},
{"tag": "driveType", "value": "All wheel drive"},
{"tag": "exteriorColor", "value": "Серый"},
{"tag": "driveType", "value": "Полный привод"},
{"tag": "engineDisplacement", "value": "2993 cc"},
{"tag": "mileage", "value": "145 000 km"},
{"tag": "year", "value": "2019"},
{"tag": "bodyType", "value": "Geländewagen"},
{"tag": "bodyType", "value": "Внедорожник"},
],
},
)
@@ -428,13 +485,13 @@ class TestMobileDeMapper(unittest.TestCase):
id="1013",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
title="BMW 320 xDrive",
raw={"attr": {"wheelDrive": "Front-wheel drive"}},
raw={"attr": {"wheelDrive": "Передний привод"}},
)
)
self.assertEqual(record.drive, "FWD")
def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None:
def test_bmw_sdrive_in_title_does_not_define_drive(self) -> None:
record = self.mapper.detail_to_car_record(
"1014",
{
@@ -443,7 +500,7 @@ class TestMobileDeMapper(unittest.TestCase):
},
)
self.assertEqual(record.drive, "2WD")
self.assertIsNone(record.drive)
def test_russian_search_values_map_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
@@ -463,6 +520,7 @@ class TestMobileDeMapper(unittest.TestCase):
"ecol": "Черный Металлик",
"tr": "Автоматическая",
"c": "Внедорожник/Пикап",
"wheelDrive": "Полный привод",
},
},
)
@@ -492,7 +550,7 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.color, "gold")
def test_truncated_white_color_maps_to_english_contract(self) -> None:
def test_german_color_is_not_mapped(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2003",
@@ -502,7 +560,7 @@ class TestMobileDeMapper(unittest.TestCase):
)
)
self.assertEqual(record.color, "white")
self.assertEqual(record.color, "other")
def test_unknown_color_maps_to_other(self) -> None:
record = self.mapper.listing_to_car_record(
@@ -538,6 +596,44 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.steering_wheel, "RIGHT")
def test_false_right_hand_drive_does_not_invent_left(self) -> None:
record = self.mapper.detail_to_car_record(
"2007",
{
"shortTitle": "BMW 320",
"rightHandDrive": False,
},
)
self.assertIsNone(record.steering_wheel)
def test_missing_steering_does_not_invent_left(self) -> None:
record = self.mapper.detail_to_car_record("2008", {"shortTitle": "BMW 320"})
self.assertIsNone(record.steering_wheel)
def test_english_and_german_enum_values_are_not_mapped(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2009",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2009",
title="BMW 320 xDrive",
raw={
"attr": {
"ecol": "Schwarz",
"tr": "Automatik",
"c": "Kombi",
"wheelDrive": "Allrad",
}
},
)
)
self.assertEqual(record.color, "other")
self.assertIsNone(record.gearbox)
self.assertEqual(record.body_type, "OTHER")
self.assertIsNone(record.drive)
def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
@@ -565,6 +661,7 @@ class TestMobileDeMapper(unittest.TestCase):
{"tag": "cubicCapacity", "value": "2 894 ccm"},
{"tag": "numberOfPreviousOwners", "value": "1"},
{"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
{"tag": "wheelDrive", "value": "Привод на четыре колеса"},
],
"images": [{"uri": first}, {"uri": second}],
"links": {"preview": unrelated},
tests/test_mobilede_requirements.py
+6 -1
View File
@@ -49,7 +49,9 @@ class _Persistence:
class TestMobileDeRequirements(unittest.TestCase):
def test_russian_transport_and_english_public_url_preserve_query(self) -> None:
search = MobileDeClient.build_search_url_from_existing(
"https://suchen.mobile.de/fahrzeuge/search.html?ms=3500&lang=en&custom=keep",
"https://www.mobile.de/ru/транспортные-средства/поиск.html"
"?isSearchRequest=true&s=Car&vc=Car"
"&ms=25100&ms=8600&ms=15200&ms=3500&ref=dsp&lang=en&custom=keep",
page_number=3,
)
transport_detail = MobileDeClient.build_transport_detail_url("123")
@@ -62,6 +64,9 @@ class TestMobileDeRequirements(unittest.TestCase):
self.assertEqual(urlsplit(public_detail).netloc, "suchen.mobile.de")
self.assertEqual(urlsplit(public_detail).path, "/fahrzeuge/details.html")
self.assertEqual(parse_qs(urlsplit(search).query)["lang"], ["ru"])
self.assertEqual(parse_qs(urlsplit(search).query)["pageNumber"], ["3"])
self.assertEqual(parse_qs(urlsplit(search).query)["ms"], ["25100", "8600", "15200", "3500"])
self.assertEqual(parse_qs(urlsplit(search).query)["ref"], ["dsp"])
self.assertEqual(parse_qs(urlsplit(search).query)["custom"], ["keep"])
self.assertEqual(parse_qs(urlsplit(transport_detail).query)["lang"], ["ru"])
self.assertEqual(parse_qs(urlsplit(public_detail).query)["lang"], ["en"])
tests/test_mobilede_scraper.py
+13 -3
View File
@@ -4,6 +4,7 @@ import unittest
from datetime import datetime, timezone
from unittest.mock import patch
from mobilede_scraper.mobile_de.client import MobileDeClient
from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
@@ -61,9 +62,9 @@ class _FakeClient:
"attributes": [
{"tag": "firstRegistration", "value": "05/2016"},
{"tag": "mileage", "value": "71 500 km"},
{"tag": "category", "value": "OffRoad"},
{"tag": "category", "value": "Внедорожник"},
{"tag": "color", "value": "Серый"},
{"tag": "wheelDrive", "value": "xDrive"},
{"tag": "wheelDrive", "value": "Полный привод"},
{"tag": "cubicCapacity", "value": "1 998 ccm"},
],
"images": ["https://img.example.test/1.jpg"],
@@ -71,6 +72,15 @@ class _FakeClient:
class TestMobileDeScraperStreamingSync(unittest.TestCase):
def test_worker_client_uses_configured_bounded_pool(self) -> None:
with patch.dict("os.environ", {"MOBILEDE_HTTP_POOL_SIZE": "7"}):
client = MobileDeClient.for_worker()
adapter = client.session.get_adapter("https://")
self.assertEqual(adapter._pool_connections, 7)
self.assertEqual(adapter._pool_maxsize, 7)
self.assertTrue(adapter._pool_block)
def test_sync_search_upserts_each_page_during_run(self) -> None:
pages = [
MobileDeSearchPage(
@@ -151,7 +161,7 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
id="2",
url="https://example.test/2",
title="BMW 320",
raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "Универсал", "ecol": "Серый", "cc": "1 998 ccm"}},
),
],
),