Files
mobile.de/tests/test_mappers.py
T

635 lines
25 KiB
Python

from __future__ import annotations
import unittest
from mobilede_scraper.mobile_de.mapper import MobileDeMapper
from mobilede_scraper.mobile_de.models import MobileDeListing
class TestMobileDeMapper(unittest.TestCase):
def setUp(self) -> None:
self.mapper = MobileDeMapper()
def test_deduplicates_images(self) -> None:
first = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
urls = [
f"{first}?rule=mo-80w",
f"{first}?rule=mo-1600",
second.replace("https:", ""),
]
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="123",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=123",
title="Honda Civic",
raw={"images": urls},
)
)
self.assertEqual(len(record.images), 2)
self.assertEqual(record.images[0].fullres_image, f"{first}?rule=mo-1600")
self.assertEqual(record.images[0].preview_image, f"{first}?rule=mo-360")
self.assertEqual(record.images[1].fullres_image, f"{second}?rule=mo-1600")
self.assertEqual(record.images[1].preview_image, f"{second}?rule=mo-360")
def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="124",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
title="Honda Civic",
raw={
"image": "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111",
"images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"}],
"mediaGallery": [
{"uri": "//img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"},
{"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444"}},
],
"trackingUrl": "https://example.test/not-an-image",
},
)
)
self.assertEqual(
[image.fullres_image for image in record.images],
[
"https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444?rule=mo-1600",
],
)
def test_replaces_existing_mobilede_image_rule(self) -> None:
source = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="125",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
title="BMW 320",
raw={"images": [{"uri": f"{source}?rule=mo-80w"}]},
)
)
self.assertEqual(record.images[0].fullres_image, f"{source}?rule=mo-1600")
self.assertEqual(record.images[0].preview_image, f"{source}?rule=mo-360")
def test_rejects_non_vehicle_mobilede_images(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="126",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=126",
title="BMW 320",
raw={
"images": [
"https://img.classistatic.de/api/v1/mo-prod/images/co2class-G?rule=mo-640",
"https://img.classistatic.de/images/logo.svg",
"https://example.test/image.jpg",
]
},
)
)
self.assertEqual(record.images, [])
def test_origin_id_uses_canonical_prefix(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="456",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=456",
title="Ford Focus",
)
)
self.assertEqual(record.origin_id, "mobile.de:456")
self.assertEqual(record.origin, "MOBILEDE")
self.assertEqual(record.selling_type, "STOCK")
self.assertFalse(record.details_confirmed)
self.assertFalse(record.images_confirmed)
self.assertTrue(record.preserve_existing_details)
def test_slug_includes_year_without_duplicate_or_trailing_hyphens(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="457",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=457",
title="Dodge -- Dart SXT!",
first_registration="2013",
)
)
self.assertEqual(record.slug, "dodge-dart-sxt-2013")
self.assertNotIn("--", record.slug)
self.assertFalse(record.slug.endswith("-"))
def test_slug_omits_year_when_year_is_absent(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="458",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=458",
title="Dodge Dart SXT",
)
)
self.assertEqual(record.slug, "dodge-dart-sxt")
def test_unknown_empty_values_and_normalization(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="999",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=999",
title="",
subtitle="",
raw={"make": {"localized": " "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}},
)
)
self.assertEqual(record.brand, "UNKNOWN")
self.assertEqual(record.model, "UNKNOWN")
self.assertIsNone(record.drive)
self.assertIsNone(record.gearbox)
# Нормализация регистра и пробелов.
record2 = self.mapper.listing_to_car_record(
MobileDeListing(
id="888",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
title="Honda Civic",
transmission=" Automatik ",
)
)
self.assertEqual(record2.gearbox, "AT")
def test_price_and_currency_parsing(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="777",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=777",
title="Toyota Corolla",
price="€ 5.200",
first_registration="05/2019",
mileage="50.100 km",
)
)
self.assertEqual(record.price, 5200)
self.assertEqual(record.currency, "EUR")
self.assertEqual(record.year, 2019)
self.assertEqual(record.mileage, 50100)
def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1001",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
title="BMW X1 xDrive",
subtitle="Sport",
raw={
"attr": {
"yc": "2016",
"cn": "DE",
"ecol": "Silber",
"c": "OffRoad",
"cc": "1 998 ccm",
}
},
)
)
self.assertEqual(record.year, 2016)
self.assertEqual(record.country, "DE")
self.assertEqual(record.color, "silver")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 1998)
self.assertEqual(record.drive, "4WD")
def test_listing_body_and_color_normalization_is_broader(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1002",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
title="BMW 320 Touring",
subtitle="EstateCar",
raw={"attr": {"ecol": "Braun"}},
)
)
self.assertEqual(record.body_type, "STATION_WAGON")
self.assertEqual(record.color, "brown")
def test_listing_maps_available_search_payload_fields(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1003",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
title="BMW 120 Frontantrieb",
raw={
"price": {"grs": {"amount": "12250"}},
"contact": {"country": "IT"},
"priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
"hasDamage": True,
"isConditionNew": True,
"cubicCapacity": "1 995 ccm",
"color": "Black Metallic",
"attr": {
"yc": "2020",
"ml": "55 000 km",
"c": "SmallCar",
"tr": "Manual",
"pvo": "1",
},
},
)
)
self.assertEqual(record.year, 2020)
self.assertEqual(record.price, 12250)
self.assertEqual(record.mileage, 55000)
self.assertEqual(record.country, "IT")
self.assertEqual(record.color, "black")
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.gearbox, "MT")
self.assertEqual(record.body_type, "HATCHBACK")
self.assertEqual(record.engine_volume, 1995)
self.assertTrue(record.one_owner)
self.assertTrue(record.new_car)
self.assertTrue(record.is_damaged)
self.assertFalse(record.repair_history)
self.assertFalse(record.non_smoking)
self.assertEqual(record.evaluation, "GOOD_PRICE")
def test_drive_mapping_uses_specific_markers_only(self) -> None:
allrad = self.mapper.listing_to_car_record(
MobileDeListing(
id="1004",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
title="BMW X3 Allrad",
)
)
unrelated = self.mapper.listing_to_car_record(
MobileDeListing(
id="1005",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
title="BMW 320 All inclusive",
)
)
self.assertEqual(allrad.drive, "4WD")
self.assertIsNone(unrelated.drive)
def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1006",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
title="BMW X3",
subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
raw={
"category": "Иное",
"attr": {
"cn": "NL",
"c": "OtherCar",
"cc": "2 993 ccm",
},
},
)
)
self.assertEqual(record.country, "NL")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 2993)
def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1007",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
title="Honda Civic",
subtitle="1.6 Automatik",
raw={"attr": {"cn": "AT", "c": "SmallCar"}},
)
)
self.assertEqual(record.country, "AT")
self.assertEqual(record.body_type, "HATCHBACK")
self.assertEqual(record.engine_volume, 1600)
def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1009",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
title="Honda Civic",
subtitle="1983",
raw={"attr": {"cn": "SK", "c": "SmallCar"}},
)
)
self.assertIsNone(record.engine_volume)
def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1010",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
title="BMW 320",
subtitle="E 46 320i 2.200 ccm 170 PS 2005",
raw={"attr": {"cn": "DE", "c": "Cabrio"}},
)
)
self.assertEqual(record.engine_volume, 2200)
def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1008",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
title="BMW 218",
raw={
"vehicle": {
"specs": {
"exteriorColor": "Silver",
"bodyType": "Cabrio",
"cubicCapacity": "1998",
"wheelDrive": "Allrad",
"transmission": "Automatic",
}
}
},
)
)
self.assertEqual(record.color, "silver")
self.assertEqual(record.body_type, "OPEN")
self.assertEqual(record.engine_volume, 1998)
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.gearbox, "AT")
def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1011",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
title="BMW 320 Touring",
subtitle="2.0 l xDrive",
raw={
"attr": {
"exteriorColor": "Skyscraper Grey Metallic",
"wheelDrive": "Antrieb vorne",
"engineDisplacement": "2.0 l",
},
"driveType": "xDrive",
},
)
)
self.assertEqual(record.color, "gray")
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.engine_volume, 2000)
def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
record = self.mapper.detail_to_car_record(
"1012",
{
"shortTitle": "BMW X3",
"subTitle": "xDrive30d",
"make": {"localized": "BMW"},
"model": {"localized": "X3"},
"price": {"grs": {"amount": "31990", "currency": "EUR"}},
"contact": {"countryCode": "DE"},
"attributes": [
{"tag": "exteriorColor", "value": "Brooklyn Grey"},
{"tag": "driveType", "value": "All wheel drive"},
{"tag": "engineDisplacement", "value": "2993 cc"},
{"tag": "mileage", "value": "145 000 km"},
{"tag": "year", "value": "2019"},
{"tag": "bodyType", "value": "Geländewagen"},
],
},
)
self.assertEqual(record.color, "gray")
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.engine_volume, 2993)
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.year, 2019)
self.assertEqual(record.origin, "MOBILEDE")
self.assertEqual(record.selling_type, "STOCK")
self.assertFalse(record.non_smoking)
self.assertFalse(record.repair_history)
self.assertTrue(record.details_confirmed)
self.assertTrue(record.images_confirmed)
self.assertFalse(record.preserve_existing_details)
def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1013",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
title="BMW 320 xDrive",
raw={"attr": {"wheelDrive": "Front-wheel drive"}},
)
)
self.assertEqual(record.drive, "FWD")
def test_bmw_sdrive_maps_to_two_wheel_drive(self) -> None:
record = self.mapper.detail_to_car_record(
"1014",
{
"shortTitle": "BMW X1",
"subTitle": "sDrive18i Advantage",
},
)
self.assertEqual(record.drive, "2WD")
def test_russian_search_values_map_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2001",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2001&lang=en",
title="Audi e-tron quattro",
raw={
"make": {"localized": "Audi"},
"model": {"localized": "e-tron"},
"price": {"grs": {"amount": 29700}},
"hasDamage": "false",
"isNew": "false",
"attr": {
"fr": "05/2021",
"ml": "59 000 км",
"ecol": "Черный Металлик",
"tr": "Автоматическая",
"c": "Внедорожник/Пикап",
},
},
)
)
self.assertEqual(record.brand, "Audi")
self.assertEqual(record.model, "e-tron")
self.assertEqual(record.year, 2021)
self.assertEqual(record.price, 29700)
self.assertEqual(record.mileage, 59000)
self.assertEqual(record.color, "black")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.body_type, "SUV")
self.assertFalse(record.is_damaged)
self.assertFalse(record.new_car)
def test_russian_gold_color_maps_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2002",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2002",
title="BMW X2",
raw={"attr": {"ecol": "Золотистый Металлик Матовый"}},
)
)
self.assertEqual(record.color, "gold")
def test_truncated_white_color_maps_to_english_contract(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2003",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2003",
title="BMW 320",
raw={"attr": {"ecol": "Wei"}},
)
)
self.assertEqual(record.color, "white")
def test_unknown_color_maps_to_other(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="2004",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=2004",
title="BMW 320",
raw={"attr": {"ecol": "Неизвестное покрытие"}},
)
)
self.assertEqual(record.color, "other")
def test_right_hand_drive_maps_from_russian_feature(self) -> None:
record = self.mapper.detail_to_car_record(
"2005",
{
"shortTitle": "BMW 320",
"features": ["Правый руль"],
},
)
self.assertEqual(record.steering_wheel, "RIGHT")
def test_right_hand_drive_maps_from_explicit_flag(self) -> None:
record = self.mapper.detail_to_car_record(
"2006",
{
"shortTitle": "BMW 320",
"rightHandDrive": True,
},
)
self.assertEqual(record.steering_wheel, "RIGHT")
def test_russian_detail_values_and_exact_gallery_map_to_english_contract(self) -> None:
first = "img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
unrelated = "img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"
record = self.mapper.detail_to_car_record(
"2002",
{
"shortTitle": "Porsche Macan",
"subTitle": "GTS",
"make": {"localized": "Porsche"},
"model": {"localized": "Macan"},
"price": {"grs": {"amount": 59900, "currency": "EUR"}},
"isConditionNew": "false",
"isDamageCase": True,
"features": [
"Привод на четыре колеса",
"Машина некурящего водителя",
],
"attributes": [
{"tag": "firstRegistration", "value": "04/2022"},
{"tag": "mileage", "value": "149 300 км"},
{"tag": "color", "value": "Синий Металлик"},
{"tag": "transmission", "value": "Автоматическая"},
{"tag": "category", "value": "Внедорожник/Пикап"},
{"tag": "cubicCapacity", "value": "2 894 ccm"},
{"tag": "numberOfPreviousOwners", "value": "1"},
{"tag": "damageCondition", "value": "Подержанный, Устраненные повреждения после ДТП"},
],
"images": [{"uri": first}, {"uri": second}],
"links": {"preview": unrelated},
},
)
self.assertEqual(record.year, 2022)
self.assertEqual(record.mileage, 149300)
self.assertEqual(record.color, "blue")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 2894)
self.assertEqual(record.drive, "4WD")
self.assertTrue(record.one_owner)
self.assertTrue(record.is_damaged)
self.assertTrue(record.non_smoking)
self.assertTrue(record.repair_history)
self.assertFalse(record.rental)
self.assertFalse(record.new_car)
self.assertEqual(len(record.images), 2)
self.assertEqual(
[image.fullres_image for image in record.images],
[
f"https://{first}?rule=mo-1600",
f"https://{second}?rule=mo-1600",
],
)
def test_detail_is_new_ad_does_not_mean_new_car(self) -> None:
record = self.mapper.detail_to_car_record(
"2003",
{
"shortTitle": "BMW X5",
"make": {"localized": "BMW"},
"model": {"localized": "X5"},
"isNew": True,
"isConditionNew": False,
"attributes": [
{"tag": "firstRegistration", "value": "05/2023"},
{"tag": "mileage", "value": "108 398 км"},
{"tag": "damageCondition", "value": "Подержанный"},
],
},
)
self.assertFalse(record.new_car)
def test_russian_mechanics_and_bmw_ix_body_are_mapped(self) -> None:
record = self.mapper.detail_to_car_record(
"2004",
{
"shortTitle": "BMW iX3",
"make": {"localized": "BMW"},
"model": {"localized": "iX3"},
"attributes": [
{"tag": "transmission", "value": "Механика"},
{"tag": "category", "value": "Иное"},
],
},
)
self.assertEqual(record.gearbox, "MT")
self.assertEqual(record.body_type, "SUV")
if __name__ == "__main__":
unittest.main()