qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use Russian transport

Browse Source
This commit is contained in:
qananasikq
2026-08-18 13:10:47 +03:00
parent 8196fe50d2
commit e6c6732ed8
4 changed files with 223 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/client.py
+28 -10
View File
@@ -17,9 +17,11 @@ from .models import MobileDeListing, MobileDeSearchPage
logger = logging.getLogger("mobile_de.client")
BASE_URL = "https://suchen.mobile.de"
SEARCH_PATH = "/fahrzeuge/search.html"
DETAIL_PATH = "/fahrzeuge/details.html"
TRANSPORT_BASE_URL = "https://www.mobile.de"
TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
PUBLIC_BASE_URL = "https://suchen.mobile.de"
PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html"
DEFAULT_HEADERS = {
"user-agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -27,7 +29,7 @@ DEFAULT_HEADERS = {
"Chrome/124.0.0.0 Safari/537.36"
),
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "en-US,en;q=0.9",
"accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7",
}
MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
@@ -103,11 +105,11 @@ class MobileDeClient:
"od": "up",
"vc": "Car",
"s": "Car",
"lang": "en",
"lang": "ru",
"pageNumber": page_number,
}
query.update({key: value for key, value in params.items() if value is not None})
return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}"
@staticmethod
def build_search_url_from_existing(
@@ -125,13 +127,26 @@ class MobileDeClient:
if page_number is not None:
query_items.append(("pageNumber", str(page_number)))
query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
query_items.append(("lang", "en"))
return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
query_items.append(("lang", "ru"))
return urlunsplit(
(
"https",
urlsplit(TRANSPORT_BASE_URL).netloc,
TRANSPORT_SEARCH_PATH,
urlencode(query_items),
"",
)
)
@staticmethod
def build_transport_detail_url(listing_id: str | int) -> str:
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"})
return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}"
@staticmethod
def build_detail_url(listing_id: str | int) -> str:
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
return f"{BASE_URL}{DETAIL_PATH}?{query}"
return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}"
def fetch_html(self, url: str, *, timeout: int = 30) -> str:
last_error: Exception | None = None
@@ -375,7 +390,10 @@ class MobileDeClient:
return ordered_pages
def fetch_detail(self, listing_id: str | int) -> dict:
html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
html = self.fetch_html(
self.build_transport_detail_url(listing_id),
timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS,
)
return extract_detail_listing(html)
@staticmethod
mobilede_scraper/mobile_de/mapper.py
+97 -14
View File
@@ -15,30 +15,42 @@ PARSER_ID_ALPHABET = ascii_letters + digits
PARSER_ID_RE = re.compile(r"^car-[A-Za-z0-9]{22}$")
_BODY_MAP = {
"кабриолет": "OPEN",
"родстер": "OPEN",
"cabrio": "OPEN",
"cabriolet": "OPEN",
"roadster": "OPEN",
"седан": "SEDAN",
"limousine": "SEDAN",
"saloon": "SEDAN",
"sedan": "SEDAN",
"внедорожник": "SUV",
"suv": "SUV",
"offroad": "SUV",
"gelandewagen": "SUV",
"geländewagen": "SUV",
"пикап": "PICKUP",
"pickup": "PICKUP",
"pick-up": "PICKUP",
"универсал": "STATION_WAGON",
"kombi": "STATION_WAGON",
"estatecar": "STATION_WAGON",
"touring": "STATION_WAGON",
"estate": "STATION_WAGON",
"минивэн": "MINIVAN",
"микроавтобус": "MINIVAN",
"фургон": "MINIVAN",
"van": "MINIVAN",
"kleinbus": "MINIVAN",
"bus": "MINIVAN",
"active tourer": "MINIVAN",
"gran tourer": "MINIVAN",
"купе": "COUPE",
"coupe": "COUPE",
"sportscar": "COUPE",
"sports car": "COUPE",
"хэтчбек": "HATCHBACK",
"малолитражный": "HATCHBACK",
"hatchback": "HATCHBACK",
"kleinwagen": "HATCHBACK",
"smallcar": "HATCHBACK",
@@ -48,13 +60,20 @@ _BODY_MAP = {
}
_GEARBOX_MAP = {
"автоматическая": "AT",
"автоматическая коробка": "AT",
"automatik": "AT",
"automatic": "AT",
"механическая": "MT",
"механическая коробка": "MT",
"manual": "MT",
"вариатор": "CVT",
"cvt": "CVT",
}
_COLOR_MAP = {
"черный": "black",
"чёрный": "black",
"schwarz": "black",
"saphirschwarz": "black",
"carbonschwarz": "black",
@@ -62,15 +81,19 @@ _COLOR_MAP = {
"jet black": "black",
"jetblack": "black",
"black": "black",
"белый": "white",
"weiss": "white",
"weiß": "white",
"alpinweiss": "white",
"alpine white": "white",
"mineralweiss": "white",
"white": "white",
"серебристый": "silver",
"серебряный": "silver",
"silber": "silver",
"argent": "silver",
"silver": "silver",
"серый": "gray",
"grau": "gray",
"grey": "gray",
"anthrazit": "gray",
@@ -83,31 +106,44 @@ _COLOR_MAP = {
"sophistograu": "gray",
"skyscraper grau": "gray",
"gray": "gray",
"красный": "red",
"rot": "red",
"burgundy": "red",
"bordeaux": "red",
"maroon": "red",
"red": "red",
"синий": "blue",
"голубой": "blue",
"blau": "blue",
"turquoise": "blue",
"cyan": "blue",
"blue": "blue",
"зеленый": "green",
"зелёный": "green",
"grün": "green",
"gruen": "green",
"green": "green",
"коричневый": "brown",
"braun": "brown",
"brown": "brown",
"бежевый": "beige",
"beige": "beige",
"champagner": "beige",
"champagne": "beige",
"creme": "beige",
"cream": "beige",
"ivory": "beige",
"желтый": "yellow",
"жёлтый": "yellow",
"gelb": "yellow",
"yellow": "yellow",
"оранжевый": "orange",
"orange": "orange",
"золотой": "gold",
"gold": "gold",
"бронзовый": "bronze",
"bronze": "bronze",
"фиолетовый": "purple",
"violett": "purple",
"lila": "purple",
"purple": "purple",
@@ -176,10 +212,7 @@ class MobileDeMapper:
or attr.get("damageCondition")
or attr.get("dc")
).lower()
is_damaged = (
bool(raw.get("hasDamage"))
or ("accident" in damage_text and "no accident" not in damage_text)
)
is_damaged = self._is_damaged_value(raw.get("hasDamage"), damage_text)
year = self._year_from_first_registration(
self._first_present(
listing.first_registration,
@@ -248,7 +281,7 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
new_car=self._bool_value(raw.get("isNew")) or self._bool_value(raw.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=listing.url,
@@ -316,9 +349,11 @@ class MobileDeMapper:
)
damage_text = self._text(attrs.get("damageCondition")).lower()
is_damaged = ("accident" in damage_text and "no accident" not in damage_text) or bool(detail.get("hasDamage"))
is_damaged = self._is_damaged_value(detail.get("hasDamage"), damage_text)
owners_text = self._text(attrs.get("numPreviousOwners"))
owners_text = self._text(
self._mapping_value(attrs, "numPreviousOwners", "numberOfPreviousOwners")
)
one_owner = self._is_one_owner(owners_text)
return CarRecord(
@@ -349,7 +384,7 @@ class MobileDeMapper:
),
selling_type="STOCK",
one_owner=one_owner,
new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
new_car=self._bool_value(detail.get("isNew")) or self._bool_value(detail.get("isConditionNew")),
is_hidden=False,
origin="MOBILEDE",
origin_url=MobileDeClient.build_detail_url(listing_id),
@@ -364,7 +399,7 @@ class MobileDeMapper:
details_confirmed=True,
images_confirmed=True,
preserve_existing_details=False,
images=self._images_from_listing(detail),
images=self._images_from_detail(detail),
)
@staticmethod
@@ -477,18 +512,18 @@ class MobileDeMapper:
@staticmethod
def _normalize_drive(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
if any(marker in text for marker in ("передний привод", "передние колеса", "front", "fwd", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
return "FWD"
if any(marker in text for marker in ("rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
if any(marker in text for marker in ("задний привод", "задние колеса", "rear", "rwd", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
return "RWD"
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
if any(marker in text for marker in ("полный привод", "все колеса", "awd", "4wd", "4x4", "quattro", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
return "4WD"
return None
@staticmethod
def _normalize_country(value: Any) -> str:
text = "" if value is None else str(value).strip().upper()
if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
if text in {"DE", "GERMANY", "DEUTSCHLAND", "ГЕРМАНИЯ"}:
return "DE"
if text in {"US", "USA", "UNITED STATES"}:
return "US"
@@ -510,7 +545,10 @@ class MobileDeMapper:
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
return "SUV"
for marker, mapped in _BODY_MAP.items():
if marker in text:
if marker in {"van", "bus"}:
if re.search(rf"\b{re.escape(marker)}\b", text):
return mapped
elif marker in text:
return mapped
return "OTHER"
@@ -597,6 +635,41 @@ class MobileDeMapper:
text = "" if value is None else str(value).strip().lower()
return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
@staticmethod
def _bool_value(value: Any) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return value != 0
text = "" if value is None else str(value).strip().lower()
if text in {"1", "true", "yes", "on", "да"}:
return True
if text in {"0", "false", "no", "off", "нет", ""}:
return False
return False
@classmethod
def _is_damaged_value(cls, flag: Any, description: Any) -> bool:
if cls._bool_value(flag):
return True
text = cls._text(description).lower()
if not text:
return False
if any(marker in text for marker in ("без дтп", "без аварий", "no accident", "accident free")):
return False
return any(
marker in text
for marker in (
"дтп",
"аварийн",
"с повреждениями",
"устраненные повреждения",
"устранённые повреждения",
"accident",
"damaged",
)
)
@staticmethod
def _rating_text(value: Any) -> str | None:
if isinstance(value, dict):
@@ -658,6 +731,16 @@ class MobileDeMapper:
for index, source_url in enumerate(source_urls)
]
@staticmethod
def _images_from_detail(raw: dict[str, Any]) -> list[ImageRecord]:
explicit_images = raw.get("images")
if isinstance(explicit_images, list) and explicit_images:
return MobileDeMapper._images_from_listing({"images": explicit_images})
media_gallery = raw.get("mediaGallery")
if isinstance(media_gallery, list) and media_gallery:
return MobileDeMapper._images_from_listing({"mediaGallery": media_gallery})
return MobileDeMapper._images_from_listing(raw)
@staticmethod
def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
if depth > 8: