382 lines
14 KiB
Python
382 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
|
|
from mobilede_scraper.mobile_de.mapper import MobileDeMapper
|
|
from mobilede_scraper.mobile_de.models import MobileDeListing
|
|
|
|
|
|
class TestMobileDeMapper(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.mapper = MobileDeMapper()
|
|
|
|
def test_deduplicates_images(self) -> None:
|
|
urls = [
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/1",
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/1",
|
|
"//img.classistatic.de/api/v1/mo-prod/images/2",
|
|
]
|
|
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="123",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=123",
|
|
title="Honda Civic",
|
|
raw={"images": urls},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(len(record.images), 2)
|
|
self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg")
|
|
self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg")
|
|
|
|
def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="124",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
|
|
title="Honda Civic",
|
|
raw={
|
|
"image": "https://img.classistatic.de/api/v1/mo-prod/images/main",
|
|
"images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}],
|
|
"mediaGallery": [
|
|
{"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"},
|
|
{"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}},
|
|
],
|
|
"trackingUrl": "https://example.test/not-an-image",
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(
|
|
[image.fullres_image for image in record.images],
|
|
[
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg",
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg",
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg",
|
|
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg",
|
|
],
|
|
)
|
|
|
|
def test_keeps_existing_mobilede_image_rule(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="125",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
|
|
title="BMW 320",
|
|
raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(
|
|
[image.fullres_image for image in record.images],
|
|
["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
|
|
)
|
|
|
|
def test_origin_id_uses_canonical_prefix(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="456",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=456",
|
|
title="Ford Focus",
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.origin_id, "mobile.de:456")
|
|
self.assertEqual(record.origin, "MOBILE_DE")
|
|
|
|
def test_unknown_empty_values_and_normalization(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="999",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=999",
|
|
title="",
|
|
subtitle="",
|
|
raw={"make": {"localized": " "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}},
|
|
)
|
|
)
|
|
self.assertEqual(record.brand, "UNKNOWN")
|
|
self.assertEqual(record.model, "UNKNOWN")
|
|
self.assertIsNone(record.drive)
|
|
self.assertIsNone(record.gearbox)
|
|
|
|
# Нормализация регистра и пробелов.
|
|
record2 = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="888",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
|
|
title="Honda Civic",
|
|
transmission=" Automatik ",
|
|
)
|
|
)
|
|
self.assertEqual(record2.gearbox, "AT")
|
|
|
|
def test_price_and_currency_parsing(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="777",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=777",
|
|
title="Toyota Corolla",
|
|
price="€ 5.200",
|
|
first_registration="05/2019",
|
|
mileage="50.100 km",
|
|
)
|
|
)
|
|
self.assertEqual(record.price, 5200)
|
|
self.assertEqual(record.currency, "EUR")
|
|
self.assertEqual(record.year, 2019)
|
|
self.assertEqual(record.mileage, 50100)
|
|
|
|
def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1001",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
|
|
title="BMW X1 xDrive",
|
|
subtitle="Sport",
|
|
raw={
|
|
"attr": {
|
|
"yc": "2016",
|
|
"cn": "DE",
|
|
"ecol": "Серебряный",
|
|
"c": "OffRoad",
|
|
"cc": "1 998 ccm",
|
|
}
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.year, 2016)
|
|
self.assertEqual(record.country, "DE")
|
|
self.assertEqual(record.color, "silver")
|
|
self.assertEqual(record.body_type, "SUV")
|
|
self.assertEqual(record.engine_volume, 1998)
|
|
self.assertEqual(record.drive, "4WD")
|
|
|
|
def test_listing_body_and_color_normalization_is_broader(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1002",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
|
|
title="BMW 320 Touring",
|
|
subtitle="EstateCar",
|
|
raw={"attr": {"ecol": "Коричневый"}},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.body_type, "STATION_WAGON")
|
|
self.assertEqual(record.color, "brown")
|
|
|
|
def test_listing_maps_available_search_payload_fields(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1003",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
|
|
title="BMW 120 Frontantrieb",
|
|
raw={
|
|
"price": {"grs": {"amount": "12250"}},
|
|
"contact": {"country": "IT"},
|
|
"priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
|
|
"hasDamage": True,
|
|
"isConditionNew": True,
|
|
"cubicCapacity": "1 995 ccm",
|
|
"color": "Black Metallic",
|
|
"attr": {
|
|
"yc": "2020",
|
|
"ml": "55 000 km",
|
|
"c": "SmallCar",
|
|
"tr": "Manual",
|
|
"pvo": "1",
|
|
},
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.year, 2020)
|
|
self.assertEqual(record.price, 12250)
|
|
self.assertEqual(record.mileage, 55000)
|
|
self.assertEqual(record.country, "IT")
|
|
self.assertEqual(record.color, "black")
|
|
self.assertEqual(record.drive, "FWD")
|
|
self.assertEqual(record.gearbox, "MT")
|
|
self.assertEqual(record.body_type, "HATCHBACK")
|
|
self.assertEqual(record.engine_volume, 1995)
|
|
self.assertTrue(record.one_owner)
|
|
self.assertTrue(record.new_car)
|
|
self.assertTrue(record.is_damaged)
|
|
self.assertTrue(record.repair_history)
|
|
self.assertEqual(record.evaluation, "GOOD_PRICE")
|
|
|
|
def test_drive_mapping_uses_specific_markers_only(self) -> None:
|
|
allrad = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1004",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
|
|
title="BMW X3 Allrad",
|
|
)
|
|
)
|
|
unrelated = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1005",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
|
|
title="BMW 320 All inclusive",
|
|
)
|
|
)
|
|
|
|
self.assertEqual(allrad.drive, "4WD")
|
|
self.assertIsNone(unrelated.drive)
|
|
|
|
def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1006",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
|
|
title="BMW X3",
|
|
subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
|
|
raw={
|
|
"category": "Иное",
|
|
"attr": {
|
|
"cn": "NL",
|
|
"c": "OtherCar",
|
|
"cc": "2 993 ccm",
|
|
},
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.country, "NL")
|
|
self.assertEqual(record.body_type, "SUV")
|
|
self.assertEqual(record.engine_volume, 2993)
|
|
|
|
def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1007",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
|
|
title="Honda Civic",
|
|
subtitle="1.6 Automatik",
|
|
raw={"attr": {"cn": "AT", "c": "SmallCar"}},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.country, "AT")
|
|
self.assertEqual(record.body_type, "HATCHBACK")
|
|
self.assertEqual(record.engine_volume, 1600)
|
|
|
|
def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1009",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
|
|
title="Honda Civic",
|
|
subtitle="1983",
|
|
raw={"attr": {"cn": "SK", "c": "SmallCar"}},
|
|
)
|
|
)
|
|
|
|
self.assertIsNone(record.engine_volume)
|
|
|
|
def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1010",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
|
|
title="BMW 320",
|
|
subtitle="E 46 320i 2.200 ccm 170 PS 2005",
|
|
raw={"attr": {"cn": "DE", "c": "Cabrio"}},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.engine_volume, 2200)
|
|
|
|
def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1008",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
|
|
title="BMW 218",
|
|
raw={
|
|
"vehicle": {
|
|
"specs": {
|
|
"exteriorColor": "Silver",
|
|
"bodyType": "Cabrio",
|
|
"cubicCapacity": "1998",
|
|
"wheelDrive": "Allrad",
|
|
"transmission": "Automatic",
|
|
}
|
|
}
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.color, "silver")
|
|
self.assertEqual(record.body_type, "OPEN")
|
|
self.assertEqual(record.engine_volume, 1998)
|
|
self.assertEqual(record.drive, "4WD")
|
|
self.assertEqual(record.gearbox, "AT")
|
|
|
|
def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1011",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
|
|
title="BMW 320 Touring",
|
|
subtitle="2.0 l xDrive",
|
|
raw={
|
|
"attr": {
|
|
"exteriorColor": "Skyscraper Grey Metallic",
|
|
"wheelDrive": "Antrieb vorne",
|
|
"engineDisplacement": "2.0 l",
|
|
},
|
|
"driveType": "xDrive",
|
|
},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.color, "gray")
|
|
self.assertEqual(record.drive, "FWD")
|
|
self.assertEqual(record.engine_volume, 2000)
|
|
|
|
def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
|
|
record = self.mapper.detail_to_car_record(
|
|
"1012",
|
|
{
|
|
"shortTitle": "BMW X3",
|
|
"subTitle": "xDrive30d",
|
|
"make": {"localized": "BMW"},
|
|
"model": {"localized": "X3"},
|
|
"price": {"grs": {"amount": "31990", "currency": "EUR"}},
|
|
"contact": {"countryCode": "DE"},
|
|
"attributes": [
|
|
{"tag": "exteriorColor", "value": "Brooklyn Grey"},
|
|
{"tag": "driveType", "value": "All wheel drive"},
|
|
{"tag": "engineDisplacement", "value": "2993 cc"},
|
|
{"tag": "mileage", "value": "145 000 km"},
|
|
{"tag": "year", "value": "2019"},
|
|
{"tag": "bodyType", "value": "Geländewagen"},
|
|
],
|
|
},
|
|
)
|
|
|
|
self.assertEqual(record.color, "gray")
|
|
self.assertEqual(record.drive, "4WD")
|
|
self.assertEqual(record.engine_volume, 2993)
|
|
self.assertEqual(record.body_type, "SUV")
|
|
self.assertEqual(record.year, 2019)
|
|
|
|
def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
|
|
record = self.mapper.listing_to_car_record(
|
|
MobileDeListing(
|
|
id="1013",
|
|
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
|
|
title="BMW 320 xDrive",
|
|
raw={"attr": {"wheelDrive": "Front-wheel drive"}},
|
|
)
|
|
)
|
|
|
|
self.assertEqual(record.drive, "FWD")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|