improve runtime sync flow
This commit is contained in:
qananasikq
@@ -4,6 +4,7 @@ import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
from .client import MobileDeClient
|
||||
@@ -11,22 +12,47 @@ from .models import MobileDeListing
|
||||
|
||||
_BODY_MAP = {
|
||||
"cabrio": "OPEN",
|
||||
"cabriolet": "OPEN",
|
||||
"кабриолет": "OPEN",
|
||||
"roadster": "OPEN",
|
||||
"кабрио": "OPEN",
|
||||
"limousine": "SEDAN",
|
||||
"saloon": "SEDAN",
|
||||
"sedan": "SEDAN",
|
||||
"сeдан": "SEDAN",
|
||||
"седан": "SEDAN",
|
||||
"suv": "SUV",
|
||||
"offroad": "SUV",
|
||||
"gelandewagen": "SUV",
|
||||
"geländewagen": "SUV",
|
||||
"pickup": "PICKUP",
|
||||
"pick-up": "PICKUP",
|
||||
"пикап": "PICKUP",
|
||||
"внедорож": "SUV",
|
||||
"kombi": "STATION_WAGON",
|
||||
"estatecar": "STATION_WAGON",
|
||||
"touring": "STATION_WAGON",
|
||||
"estate": "STATION_WAGON",
|
||||
"универсал": "STATION_WAGON",
|
||||
"van": "MINIVAN",
|
||||
"kleinbus": "MINIVAN",
|
||||
"bus": "MINIVAN",
|
||||
"active tourer": "MINIVAN",
|
||||
"gran tourer": "MINIVAN",
|
||||
"минивэн": "MINIVAN",
|
||||
"фургон": "MINIVAN",
|
||||
"coupe": "COUPE",
|
||||
"sportscar": "COUPE",
|
||||
"sports car": "COUPE",
|
||||
"купе": "COUPE",
|
||||
"hatchback": "HATCHBACK",
|
||||
"kleinwagen": "HATCHBACK",
|
||||
"кляйнваген": "HATCHBACK",
|
||||
"малолитраж": "HATCHBACK",
|
||||
"хэтч": "HATCHBACK",
|
||||
"smallcar": "HATCHBACK",
|
||||
"small car": "HATCHBACK",
|
||||
"compact": "HATCHBACK",
|
||||
"compactcar": "HATCHBACK",
|
||||
}
|
||||
|
||||
_GEARBOX_MAP = {
|
||||
@@ -40,30 +66,87 @@ _GEARBOX_MAP = {
|
||||
|
||||
_COLOR_MAP = {
|
||||
"schwarz": "black",
|
||||
"saphirschwarz": "black",
|
||||
"carbonschwarz": "black",
|
||||
"obsidianschwarz": "black",
|
||||
"jet black": "black",
|
||||
"jetblack": "black",
|
||||
"черн": "black",
|
||||
"black": "black",
|
||||
"weiss": "white",
|
||||
"weiß": "white",
|
||||
"alpinweiss": "white",
|
||||
"alpine white": "white",
|
||||
"mineralweiss": "white",
|
||||
"бел": "white",
|
||||
"white": "white",
|
||||
"grau": "gray",
|
||||
"сер": "gray",
|
||||
"gray": "gray",
|
||||
"silber": "silver",
|
||||
"argent": "silver",
|
||||
"сереб": "silver",
|
||||
"silver": "silver",
|
||||
"grau": "gray",
|
||||
"grey": "gray",
|
||||
"anthrazit": "gray",
|
||||
"anthracite": "gray",
|
||||
"graphit": "gray",
|
||||
"graphite": "gray",
|
||||
"spacegrau": "gray",
|
||||
"brooklyn grau": "gray",
|
||||
"brooklyn grey": "gray",
|
||||
"sophistograu": "gray",
|
||||
"skyscraper grau": "gray",
|
||||
"сер": "gray",
|
||||
"gray": "gray",
|
||||
"rot": "red",
|
||||
"burgundy": "red",
|
||||
"bordeaux": "red",
|
||||
"maroon": "red",
|
||||
"красн": "red",
|
||||
"red": "red",
|
||||
"blau": "blue",
|
||||
"turquoise": "blue",
|
||||
"cyan": "blue",
|
||||
"син": "blue",
|
||||
"blue": "blue",
|
||||
"grün": "green",
|
||||
"gruen": "green",
|
||||
"зелен": "green",
|
||||
"green": "green",
|
||||
"braun": "brown",
|
||||
"корич": "brown",
|
||||
"brown": "brown",
|
||||
"beige": "beige",
|
||||
"champagner": "beige",
|
||||
"champagne": "beige",
|
||||
"creme": "beige",
|
||||
"cream": "beige",
|
||||
"ivory": "beige",
|
||||
"беж": "beige",
|
||||
"gelb": "yellow",
|
||||
"желт": "yellow",
|
||||
"yellow": "yellow",
|
||||
"orange": "orange",
|
||||
"оранж": "orange",
|
||||
"gold": "gold",
|
||||
"золот": "gold",
|
||||
"bronze": "bronze",
|
||||
"бронз": "bronze",
|
||||
"violett": "purple",
|
||||
"lila": "purple",
|
||||
"фиолет": "purple",
|
||||
"purple": "purple",
|
||||
}
|
||||
|
||||
_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
|
||||
_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
|
||||
_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
|
||||
_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
|
||||
_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
|
||||
_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
|
||||
_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
|
||||
_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
|
||||
_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
|
||||
|
||||
|
||||
class MobileDeMapper:
|
||||
"""Map mobile.de payloads into CarRecord."""
|
||||
@@ -78,35 +161,124 @@ class MobileDeMapper:
|
||||
model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
|
||||
origin_id = self.origin_id(str(listing.id))
|
||||
title = " ".join(part for part in [listing.title, listing.subtitle] if part)
|
||||
contact = raw.get("contact") if isinstance(raw.get("contact"), dict) else {}
|
||||
nested_country = self._find_first_value(raw, _COUNTRY_VALUE_KEYS)
|
||||
nested_color = self._find_first_value(raw, _COLOR_VALUE_KEYS)
|
||||
nested_body = self._find_first_value(raw, _BODY_VALUE_KEYS)
|
||||
nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
|
||||
nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
|
||||
nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
|
||||
attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
|
||||
attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
|
||||
attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
|
||||
attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
|
||||
attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
|
||||
attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
|
||||
drive_text = " ".join(
|
||||
part
|
||||
for part in [
|
||||
listing.title,
|
||||
listing.subtitle,
|
||||
self._mapping_value(attr, "an"),
|
||||
attr_drive,
|
||||
raw.get("wheelDrive"),
|
||||
raw.get("drivetrain"),
|
||||
raw.get("drive"),
|
||||
raw.get("driveType"),
|
||||
raw.get("modelDescription"),
|
||||
raw.get("variant"),
|
||||
raw.get("trim"),
|
||||
nested_drive,
|
||||
]
|
||||
if isinstance(part, str) and part.strip()
|
||||
)
|
||||
damage_text = self._text(
|
||||
raw.get("damageCondition")
|
||||
or attr.get("damageCondition")
|
||||
or attr.get("dc")
|
||||
).lower()
|
||||
is_damaged = (
|
||||
bool(raw.get("hasDamage"))
|
||||
or ("дтп" in damage_text and "без дтп" not in damage_text)
|
||||
or ("accident" in damage_text and "no accident" not in damage_text)
|
||||
)
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._parser_id(origin_id),
|
||||
brand=brand[:50] or "UNKNOWN",
|
||||
model=model[:50] or "UNKNOWN",
|
||||
year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
|
||||
price=self._money_to_int(listing.price or raw.get("p")),
|
||||
year=self._year_from_first_registration(
|
||||
self._first_present(
|
||||
listing.first_registration,
|
||||
attr.get("fr"),
|
||||
attr.get("yc"),
|
||||
raw.get("firstRegistration"),
|
||||
raw.get("firstRegistrationYear"),
|
||||
raw.get("year"),
|
||||
)
|
||||
),
|
||||
price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
|
||||
currency="EUR",
|
||||
mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
|
||||
country=self._normalize_country("DE"),
|
||||
mileage=self._int_from_text(
|
||||
self._first_present(listing.mileage, attr.get("ml"), raw.get("mileage"))
|
||||
) or 0,
|
||||
country=self._normalize_country(
|
||||
self._first_present(
|
||||
attr_country,
|
||||
contact.get("countryCode"),
|
||||
contact.get("country"),
|
||||
raw.get("countryCode"),
|
||||
raw.get("country"),
|
||||
nested_country,
|
||||
"DE",
|
||||
)
|
||||
),
|
||||
is_sold=False,
|
||||
color=self._normalize_color(attr.get("ecol")),
|
||||
drive=None,
|
||||
gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
|
||||
color=self._normalize_color(
|
||||
self._first_present(
|
||||
attr_color,
|
||||
raw.get("color"),
|
||||
raw.get("manufacturerColorName"),
|
||||
raw.get("exteriorColor"),
|
||||
nested_color,
|
||||
)
|
||||
),
|
||||
drive=self._normalize_drive(drive_text),
|
||||
gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
|
||||
steering_wheel="LEFT",
|
||||
body_type=self._normalize_body(attr.get("c")),
|
||||
engine_volume=self._int_from_text(attr.get("cc")),
|
||||
body_type=self._normalize_body_from_candidates(
|
||||
attr_body,
|
||||
raw.get("category"),
|
||||
raw.get("bodyType"),
|
||||
nested_body,
|
||||
listing.subtitle,
|
||||
listing.title,
|
||||
),
|
||||
engine_volume=self._engine_volume_from_candidates(
|
||||
attr_engine,
|
||||
raw.get("cubicCapacity"),
|
||||
raw.get("cc"),
|
||||
raw.get("engineVolume"),
|
||||
raw.get("displacement"),
|
||||
raw.get("engineDisplacement"),
|
||||
nested_engine,
|
||||
raw.get("modelDescription"),
|
||||
raw.get("variant"),
|
||||
listing.subtitle,
|
||||
listing.title,
|
||||
),
|
||||
selling_type="CLASSIFIED",
|
||||
one_owner=(str(attr.get("pvo") or "").strip() == "1"),
|
||||
new_car=False,
|
||||
one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
|
||||
new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
|
||||
is_hidden=False,
|
||||
origin="MOBILE_DE",
|
||||
origin_url=listing.url,
|
||||
origin_id=origin_id,
|
||||
is_damaged=bool(raw.get("hasDamage")),
|
||||
evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
|
||||
is_damaged=is_damaged,
|
||||
evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=bool(raw.get("hasDamage")),
|
||||
repair_history=is_damaged,
|
||||
slug=self._slugify(title or f"{brand} {model}"),
|
||||
last_seen_at=datetime.now(timezone.utc),
|
||||
images=self._images_from_listing(raw),
|
||||
@@ -127,17 +299,45 @@ class MobileDeMapper:
|
||||
origin_id = self.origin_id(str(listing_id))
|
||||
|
||||
price_amount, price_currency = self._detail_price(detail.get("price"))
|
||||
mileage = self._int_from_text(attrs.get("mileage")) or 0
|
||||
year = self._year_from_first_registration(attrs.get("firstRegistration"))
|
||||
gearbox = self._normalize_gearbox(attrs.get("transmission"))
|
||||
body_type = self._normalize_body(attrs.get("category") or detail.get("category"))
|
||||
color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName"))
|
||||
mileage = self._int_from_text(self._mapping_value(attrs, "mileage")) or 0
|
||||
year = self._year_from_first_registration(self._mapping_value(attrs, "firstRegistration", "year", "registrationDate"))
|
||||
gearbox = self._normalize_gearbox(self._first_present(self._mapping_value(attrs, "transmission", "gearbox", "transmissionType"), detail.get("transmission"), detail.get("gearbox")))
|
||||
body_type = self._normalize_body_from_candidates(
|
||||
self._mapping_value(attrs, "category", "bodyType", "body", "vehicleCategory"),
|
||||
detail.get("category"),
|
||||
detail.get("bodyType"),
|
||||
subtitle,
|
||||
short_title,
|
||||
)
|
||||
color = self._normalize_color(
|
||||
self._first_present(
|
||||
self._mapping_value(attrs, "color", "exteriorColor", "manufacturerColorName", "paint"),
|
||||
detail.get("color"),
|
||||
detail.get("manufacturerColorName"),
|
||||
detail.get("exteriorColor"),
|
||||
self._find_first_value(detail, _COLOR_VALUE_KEYS),
|
||||
)
|
||||
)
|
||||
drive_text = " ".join(
|
||||
part
|
||||
for part in [
|
||||
short_title,
|
||||
subtitle,
|
||||
self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
|
||||
detail.get("wheelDrive"),
|
||||
detail.get("drivetrain"),
|
||||
detail.get("drive"),
|
||||
detail.get("driveType"),
|
||||
self._find_first_value(detail, _DRIVE_VALUE_KEYS),
|
||||
]
|
||||
if isinstance(part, str) and part.strip()
|
||||
)
|
||||
|
||||
damage_text = self._text(attrs.get("damageCondition")).lower()
|
||||
is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
|
||||
|
||||
owners_text = self._text(attrs.get("numPreviousOwners"))
|
||||
one_owner = owners_text in {"1", "01", "1.0"}
|
||||
one_owner = self._is_one_owner(owners_text)
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._parser_id(origin_id),
|
||||
@@ -150,11 +350,21 @@ class MobileDeMapper:
|
||||
country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
|
||||
is_sold=False,
|
||||
color=color,
|
||||
drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")),
|
||||
drive=self._normalize_drive(drive_text),
|
||||
gearbox=gearbox,
|
||||
steering_wheel="LEFT",
|
||||
body_type=body_type,
|
||||
engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")),
|
||||
engine_volume=self._engine_volume_from_candidates(
|
||||
self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
|
||||
detail.get("cubicCapacity"),
|
||||
detail.get("cc"),
|
||||
detail.get("engineVolume"),
|
||||
detail.get("displacement"),
|
||||
detail.get("engineDisplacement"),
|
||||
self._find_first_value(detail, _ENGINE_VOLUME_VALUE_KEYS),
|
||||
subtitle,
|
||||
short_title,
|
||||
),
|
||||
selling_type="CLASSIFIED",
|
||||
one_owner=one_owner,
|
||||
new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
|
||||
@@ -163,7 +373,7 @@ class MobileDeMapper:
|
||||
origin_url=MobileDeClient.build_detail_url(listing_id),
|
||||
origin_id=origin_id,
|
||||
is_damaged=is_damaged,
|
||||
evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None,
|
||||
evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=is_damaged,
|
||||
@@ -185,6 +395,53 @@ class MobileDeMapper:
|
||||
def _text(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
@staticmethod
|
||||
def _first_present(*values: Any) -> Any:
|
||||
for value in values:
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, str) and not value.strip():
|
||||
continue
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalized_key(value: Any) -> str:
|
||||
return re.sub(r"[^a-z0-9]", "", str(value or "").lower())
|
||||
|
||||
@classmethod
|
||||
def _find_first_value(cls, value: Any, keys: set[str], *, depth: int = 0) -> Any:
|
||||
if depth > 6:
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
if cls._normalized_key(key) in keys and cls._first_present(item) is not None:
|
||||
if not isinstance(item, (dict, list)):
|
||||
return item
|
||||
for item in value.values():
|
||||
found = cls._find_first_value(item, keys, depth=depth + 1)
|
||||
if cls._first_present(found) is not None:
|
||||
return found
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found = cls._find_first_value(item, keys, depth=depth + 1)
|
||||
if cls._first_present(found) is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _mapping_value(cls, mapping: Any, *keys: str) -> Any:
|
||||
if not isinstance(mapping, dict):
|
||||
return None
|
||||
for key in keys:
|
||||
if key in mapping and cls._first_present(mapping.get(key)) is not None:
|
||||
return mapping.get(key)
|
||||
normalized_keys = {cls._normalized_key(key) for key in keys if key}
|
||||
for existing_key, value in mapping.items():
|
||||
if cls._normalized_key(existing_key) in normalized_keys and cls._first_present(value) is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _money_to_int(cls, value: Any) -> int | None:
|
||||
if isinstance(value, dict):
|
||||
@@ -231,11 +488,11 @@ class MobileDeMapper:
|
||||
@staticmethod
|
||||
def _normalize_drive(value: Any) -> str | None:
|
||||
text = "" if value is None else str(value).lower()
|
||||
if any(marker in text for marker in ("front", "fwd", "перед")):
|
||||
if any(marker in text for marker in ("front", "fwd", "перед", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
|
||||
return "FWD"
|
||||
if any(marker in text for marker in ("rear", "rwd", "зад")):
|
||||
if any(marker in text for marker in ("rear", "rwd", "зад", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
|
||||
return "RWD"
|
||||
if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")):
|
||||
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "полный", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
|
||||
return "4WD"
|
||||
return None
|
||||
|
||||
@@ -254,24 +511,116 @@ class MobileDeMapper:
|
||||
return "JP"
|
||||
if text in {"KR", "KOREA", "SOUTH KOREA"}:
|
||||
return "KR"
|
||||
if re.fullmatch(r"[A-Z]{2}", text):
|
||||
return text
|
||||
return "NA"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_body(value: Any) -> str:
|
||||
text = "" if value is None else str(value).lower()
|
||||
text = text.replace("ё", "е")
|
||||
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
|
||||
return "SUV"
|
||||
for marker, mapped in _BODY_MAP.items():
|
||||
if marker in text:
|
||||
return mapped
|
||||
return "OTHER"
|
||||
|
||||
@classmethod
|
||||
def _normalize_body_from_candidates(cls, *values: Any) -> str:
|
||||
for value in values:
|
||||
normalized = cls._normalize_body(value)
|
||||
if normalized != "OTHER":
|
||||
return normalized
|
||||
return "OTHER"
|
||||
|
||||
@classmethod
|
||||
def _engine_volume_from_candidates(cls, *values: Any) -> int | None:
|
||||
fallback_texts: list[str] = []
|
||||
free_text_start = max(0, len(values) - 2)
|
||||
for index, value in enumerate(values):
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
continue
|
||||
fallback_texts.append(text)
|
||||
if index < free_text_start and re.fullmatch(r"[\d\s.,]+", text):
|
||||
parsed = cls._int_from_text(text)
|
||||
if parsed and 500 <= parsed <= 9000:
|
||||
return parsed
|
||||
if re.search(r"(ccm|cm3|cm³|\bcc\b|куб|cubic|displacement)", text, re.IGNORECASE):
|
||||
parsed = cls._engine_volume_from_cc_text(text)
|
||||
if parsed:
|
||||
return parsed
|
||||
if re.search(r"\b(?:liter|litre|l)\b", text, re.IGNORECASE):
|
||||
parsed = cls._engine_volume_from_liter_text(text)
|
||||
if parsed:
|
||||
return parsed
|
||||
continue
|
||||
parsed = cls._int_from_text(value)
|
||||
if parsed and 500 <= parsed <= 9000:
|
||||
return parsed
|
||||
|
||||
for text in fallback_texts:
|
||||
match = re.search(r"(?<![\w])([1-6])[\.,](\d{1,2})(?!\d)", text.lower())
|
||||
if match:
|
||||
liters = float(f"{match.group(1)}.{match.group(2)}")
|
||||
return int(round(liters * 1000))
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _engine_volume_from_cc_text(cls, value: str) -> int | None:
|
||||
text = str(value or "")
|
||||
match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc|куб)", text, re.IGNORECASE)
|
||||
if not match:
|
||||
return None
|
||||
parsed = cls._int_from_text(match.group(1))
|
||||
if parsed and 500 <= parsed <= 9000:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _engine_volume_from_liter_text(cls, value: str) -> int | None:
|
||||
text = str(value or "")
|
||||
match = re.search(r"(?<!\d)([1-8])(?:[\.,](\d{1,2}))?\s*(?:l|liter|litre)(?![a-z])", text, re.IGNORECASE)
|
||||
if not match:
|
||||
return None
|
||||
decimals = (match.group(2) or "0").ljust(1, "0")
|
||||
liters = float(f"{match.group(1)}.{decimals}")
|
||||
parsed = int(round(liters * 1000))
|
||||
if 500 <= parsed <= 9000:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: Any) -> str:
|
||||
text = "" if value is None else str(value).lower().strip()
|
||||
text = text.replace("ё", "е")
|
||||
text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
|
||||
text = re.sub(r"[_\-/]+", " ", text)
|
||||
for marker, mapped in _COLOR_MAP.items():
|
||||
if marker in text:
|
||||
return mapped
|
||||
return text[:50] if text else "other"
|
||||
|
||||
@staticmethod
|
||||
def _is_one_owner(value: Any) -> bool:
|
||||
text = "" if value is None else str(value).strip().lower()
|
||||
return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
|
||||
|
||||
@staticmethod
|
||||
def _rating_text(value: Any) -> str | None:
|
||||
if isinstance(value, dict):
|
||||
for key in ("rating", "ratingLabel", "label", "value"):
|
||||
text = MobileDeMapper._text(value.get(key))
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
text = MobileDeMapper._text(value)
|
||||
return text or None
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
|
||||
@@ -306,38 +655,69 @@ class MobileDeMapper:
|
||||
|
||||
@staticmethod
|
||||
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
|
||||
urls: list[str] = []
|
||||
image = raw.get("image")
|
||||
if isinstance(image, str):
|
||||
urls.append(MobileDeMapper._normalize_image_url(image))
|
||||
images = raw.get("images")
|
||||
if isinstance(images, list):
|
||||
for item in images:
|
||||
if isinstance(item, str):
|
||||
urls.append(MobileDeMapper._normalize_image_url(item))
|
||||
elif isinstance(item, dict):
|
||||
src = item.get("src") or item.get("url") or item.get("uri")
|
||||
if src:
|
||||
urls.append(MobileDeMapper._normalize_image_url(str(src)))
|
||||
media_gallery = raw.get("mediaGallery")
|
||||
if isinstance(media_gallery, list):
|
||||
for item in media_gallery:
|
||||
if isinstance(item, dict):
|
||||
src = item.get("uri") or item.get("url")
|
||||
if src:
|
||||
urls.append(MobileDeMapper._normalize_image_url(str(src)))
|
||||
urls = MobileDeMapper._extract_image_urls(raw)
|
||||
return [
|
||||
ImageRecord(fullres_image=url, preview_image=url, order_index=index)
|
||||
for index, url in enumerate(dict.fromkeys(url for url in urls if url))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
|
||||
if depth > 8:
|
||||
return []
|
||||
urls: list[str] = []
|
||||
parent_hint = MobileDeMapper._has_image_field_hint(parent_key)
|
||||
if isinstance(value, str):
|
||||
if parent_hint or MobileDeMapper._looks_like_image_url(value):
|
||||
normalized = MobileDeMapper._normalize_image_url(value)
|
||||
if MobileDeMapper._looks_like_image_url(normalized):
|
||||
urls.append(normalized)
|
||||
return urls
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=parent_key, depth=depth + 1))
|
||||
return urls
|
||||
if isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
key_text = str(key or "")
|
||||
urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=key_text, depth=depth + 1))
|
||||
return urls
|
||||
return urls
|
||||
|
||||
@staticmethod
|
||||
def _has_image_field_hint(value: str) -> bool:
|
||||
normalized = re.sub(r"[^a-z0-9]", "", str(value or "").lower())
|
||||
return normalized in _IMAGE_FIELD_HINTS or any(hint in normalized for hint in _IMAGE_FIELD_HINTS)
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_image_url(value: str) -> bool:
|
||||
text = str(value or "").strip().lower()
|
||||
if not text:
|
||||
return False
|
||||
if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
|
||||
return False
|
||||
return any(marker in text for marker in _IMAGE_URL_MARKERS)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_image_url(value: str) -> str:
|
||||
url = str(value).strip()
|
||||
if not url:
|
||||
return ""
|
||||
if url.startswith("//"):
|
||||
return f"https:{url}"
|
||||
if url.startswith("http://") or url.startswith("https://"):
|
||||
url = f"https:{url}"
|
||||
elif not (url.startswith("http://") or url.startswith("https://")):
|
||||
url = f"https://{url.lstrip('/')}"
|
||||
return MobileDeMapper._normalize_mobilede_image_rule(url)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_mobilede_image_rule(url: str) -> str:
|
||||
parsed = urlsplit(url)
|
||||
if "img.classistatic.de" not in parsed.netloc.lower():
|
||||
return url
|
||||
return f"https://{url.lstrip('/')}"
|
||||
if "/api/v1/mo-prod/images/" not in parsed.path:
|
||||
return url
|
||||
query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
|
||||
if any(key.lower() == "rule" for key, _value in query_pairs):
|
||||
return url
|
||||
query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))
|
||||
|
||||
Reference in New Issue
Block a user