from __future__ import annotations import logging import os import random import threading import time from concurrent.futures import ThreadPoolExecutor, as_completed from collections.abc import Callable, Iterable from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import requests from ..core.config import ProxyConfig from .flight import extract_detail_listing, extract_search_card_enrichment, extract_search_results from .models import MobileDeListing, MobileDeSearchPage logger = logging.getLogger("mobile_de.client") TRANSPORT_BASE_URL = "https://www.mobile.de" TRANSPORT_SEARCH_PATH = "/ru/транспортные-средства/поиск.html" TRANSPORT_DETAIL_PATH = "/ru/транспортные-средства/подробности.html" LEGACY_DETAIL_PATH = "/fahrzeuge/details.html" PUBLIC_BASE_URL = "https://suchen.mobile.de" PUBLIC_DETAIL_PATH = "/fahrzeuge/details.html" DEFAULT_HEADERS = { "user-agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/124.0.0.0 Safari/537.36" ), "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "accept-language": "ru-RU,ru;q=0.9,de;q=0.8,en;q=0.7", } MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4"))) MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2"))) MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20"))) MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5"))) MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504} MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15")))) MOBILEDE_DETAIL_TRANSPORT = os.getenv("MOBILEDE_DETAIL_TRANSPORT", "flight").strip().lower() class MobileDeDetailError(RuntimeError): def __init__( self, message: str, *, error_kind: str, status_code: int | None = None, ) -> None: super().__init__(message) self.error_kind = error_kind self.status_code = status_code class MobileDeDetailParseError(MobileDeDetailError, ValueError): pass def classify_detail_error(exc: Exception) -> tuple[str, int | None]: status_code = getattr(exc, "status_code", None) if status_code is None: status_code = getattr(getattr(exc, "response", None), "status_code", None) normalized_status = int(status_code) if status_code is not None else None explicit_kind = getattr(exc, "error_kind", None) if explicit_kind: return str(explicit_kind), normalized_status if normalized_status in {404, 410}: return "unavailable", normalized_status if normalized_status == 403: return "blocked", normalized_status if normalized_status == 429: return "throttled", normalized_status if normalized_status is not None and normalized_status >= 500: return "transient", normalized_status if isinstance(exc, requests.RequestException): return "network", normalized_status if isinstance(exc, (ValueError, KeyError, TypeError)): return "parse_error", normalized_status return "transient", normalized_status class MobileDeClient: """HTTP client for mobile.de search/detail pages.""" def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None: self.session = session or requests.Session() self.session.headers.update(DEFAULT_HEADERS) self.delay_seconds = max(0.0, delay_seconds) self.proxy_config = ProxyConfig() proxies = self.proxy_config.to_requests_proxies() if proxies: self.session.proxies.update(proxies) @staticmethod def _is_retryable_status(status_code: int) -> bool: return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES @staticmethod def _compute_backoff(attempt: int) -> float: base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1)) bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0 return max(0.0, bounded + jitter) @classmethod def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient": session = requests.Session() pool_size = max(1, int(os.getenv("MOBILEDE_HTTP_POOL_SIZE", "16"))) adapter = requests.adapters.HTTPAdapter( pool_connections=pool_size, pool_maxsize=pool_size, max_retries=0, pool_block=True, ) session.mount("https://", adapter) session.mount("http://", adapter) client = cls(session=session, delay_seconds=delay_seconds) if client.proxy_config.enabled: logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server) return client @staticmethod def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str: make = str(make_id).strip() model = str(model_id).strip() if model_id is not None else "" return f"{make};{model};;" @staticmethod def build_search_url(page_number: int = 1, **params: str | int | None) -> str: query = { "sb": "rel", "od": "up", "vc": "Car", "s": "Car", "lang": "ru", "pageNumber": page_number, } query.update({key: value for key, value in params.items() if value is not None}) return f"{TRANSPORT_BASE_URL}{TRANSPORT_SEARCH_PATH}?{urlencode(query)}" @staticmethod def build_search_url_from_existing( search_url: str, *, page_number: int | None = None, **params: str | int | None, ) -> str: parts = urlsplit(search_url) query_items = [ (key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "pageNumber" and key != "lang" and key not in params ] if page_number is not None: query_items.append(("pageNumber", str(page_number))) query_items.extend((key, str(value)) for key, value in params.items() if value is not None) query_items.append(("lang", "ru")) return urlunsplit( ( "https", urlsplit(TRANSPORT_BASE_URL).netloc, TRANSPORT_SEARCH_PATH, urlencode(query_items), "", ) ) @staticmethod def build_transport_detail_url(listing_id: str | int) -> str: query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "ru"}) return f"{TRANSPORT_BASE_URL}{TRANSPORT_DETAIL_PATH}?{query}" @staticmethod def build_legacy_detail_url(listing_id: str | int) -> str: query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "de"}) return f"{TRANSPORT_BASE_URL}{LEGACY_DETAIL_PATH}?{query}" @staticmethod def build_detail_url(listing_id: str | int) -> str: query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"}) return f"{PUBLIC_BASE_URL}{PUBLIC_DETAIL_PATH}?{query}" def fetch_html(self, url: str, *, timeout: int = 30, max_retries: int | None = None) -> str: last_error: Exception | None = None retries = MOBILEDE_HTTP_MAX_RETRIES if max_retries is None else max(0, int(max_retries)) attempts = max(1, retries + 1) for attempt in range(1, attempts + 1): try: response = self.session.get(url, timeout=timeout) if self._is_retryable_status(response.status_code): if attempt < attempts: sleep_seconds = self._compute_backoff(attempt) logger.warning( "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s", response.status_code, attempt, attempts, sleep_seconds, url, ) if sleep_seconds: time.sleep(sleep_seconds) continue response.raise_for_status() return response.text except requests.RequestException as exc: last_error = exc status_code = getattr(getattr(exc, "response", None), "status_code", None) retryable = isinstance(exc, (requests.ConnectionError, requests.Timeout)) or bool( status_code is not None and self._is_retryable_status(int(status_code)) ) if attempt >= attempts or not retryable: raise sleep_seconds = self._compute_backoff(attempt) logger.warning( "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s", attempt, attempts, status_code, sleep_seconds, url, exc, ) if sleep_seconds: time.sleep(sleep_seconds) if last_error is not None: raise last_error raise RuntimeError("mobile.de fetch_html failed without a captured exception") def fetch_search_page( self, page_number: int = 1, *, search_url: str | None = None, timeout: int = 30, max_retries: int | None = None, **params: str | int | None, ) -> MobileDeSearchPage: url = ( self.build_search_url_from_existing(search_url, page_number=page_number, **params) if search_url else self.build_search_url(page_number=page_number, **params) ) html = self.fetch_html(url, timeout=timeout, max_retries=max_retries) raw = extract_search_results(html) ssr_enrichment_by_id = extract_search_card_enrichment(html) raw_listings = [ self._merge_search_card_enrichment(item, ssr_enrichment_by_id) for item in raw.get("listings", []) if isinstance(item, dict) ] listings = [self._map_listing(item) for item in raw_listings] return MobileDeSearchPage( url=url, page_number=page_number, total_results=raw.get("numResultsTotal"), listings=listings, raw_search_results=raw, ) @staticmethod def _merge_search_card_enrichment( item: dict, enrichment_by_id: dict[str, dict[str, str]], ) -> dict: enriched = dict(item) listing_id = str(enriched.get("id") or enriched.get("adId") or "") enrichment = enrichment_by_id.get(listing_id) if not enrichment: return enriched enriched.update(enrichment) if not enriched.get("shortTitle"): enriched["shortTitle"] = enrichment.get("searchSsrTitle") if not enriched.get("subTitle"): enriched["subTitle"] = enrichment.get("searchSsrSubtitle") return enriched def iter_search_pages( self, *, start_page: int = 1, max_pages: int | None = None, search_url: str | None = None, stop_after_empty: bool = True, progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, **params: str | int | None, ) -> Iterable[MobileDeSearchPage]: page_number = start_page pages_seen = 0 logger.debug( "mobile.de search window started: start_page=%s max_pages=%s params=%s", start_page, max_pages, {key: value for key, value in params.items() if value is not None}, ) while max_pages is None or pages_seen < max_pages: logger.debug("mobile.de fetching search page=%s", page_number) page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params) page_meta = { "page_number": page.page_number, "pages_seen": pages_seen + 1, "max_pages": max_pages, "listing_count": len(page.listings), "total_results": page.total_results, } logger.debug( "mobile.de fetched search page=%s listings=%s total_results=%s", page.page_number, len(page.listings), page.total_results, ) if progress_callback is not None: progress_callback(page, page_meta) if stop_after_empty and not page.listings: logger.debug("mobile.de stopping search window on empty page=%s", page.page_number) break yield page pages_seen += 1 page_number += 1 if self.delay_seconds: time.sleep(self.delay_seconds) logger.debug( "mobile.de search window finished: pages_seen=%s next_page=%s", pages_seen, page_number, ) def fetch_search_pages_concurrent( self, *, start_page: int = 1, max_pages: int = 1, workers: int = 8, search_url: str | None = None, stop_after_empty: bool = True, progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, **params: str | int | None, ) -> list[MobileDeSearchPage]: max_pages = max(1, int(max_pages)) workers = max(1, min(int(workers), max_pages)) first_page_number = max(1, int(start_page)) page_numbers = list(range(first_page_number, first_page_number + max_pages)) pages_by_number: dict[int, MobileDeSearchPage] = {} thread_local = threading.local() request_schedule_lock = threading.Lock() next_request_at = 0.0 def _fetch_page(page_number: int) -> MobileDeSearchPage: nonlocal next_request_at client = getattr(thread_local, "client", None) if client is None: client = MobileDeClient.for_worker(delay_seconds=0) thread_local.client = client if self.delay_seconds: with request_schedule_lock: now = time.monotonic() scheduled_at = max(now, next_request_at) next_request_at = scheduled_at + self.delay_seconds sleep_seconds = scheduled_at - now if sleep_seconds > 0: time.sleep(sleep_seconds) return client.fetch_search_page(page_number=page_number, search_url=search_url, **params) first_page = _fetch_page(first_page_number) pages_by_number[first_page_number] = first_page if progress_callback is not None: progress_callback( first_page, { "page_number": first_page.page_number, "pages_seen": 1, "max_pages": max_pages, "listing_count": len(first_page.listings), "total_results": first_page.total_results, }, ) if first_page_number == 1 and first_page.total_results is not None: results_per_page = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20"))) available_pages = max(1, (int(first_page.total_results) + results_per_page - 1) // results_per_page) page_numbers = page_numbers[: min(len(page_numbers), available_pages)] remaining_page_numbers = page_numbers[1:] with ThreadPoolExecutor(max_workers=workers) as executor: futures = { executor.submit(_fetch_page, page_number): page_number for page_number in remaining_page_numbers } for future in as_completed(futures): page_number = futures[future] page = future.result() pages_by_number[page_number] = page if progress_callback is not None: progress_callback( page, { "page_number": page.page_number, "pages_seen": len(pages_by_number), "max_pages": max_pages, "listing_count": len(page.listings), "total_results": page.total_results, }, ) ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number] if stop_after_empty: non_empty_pages: list[MobileDeSearchPage] = [] for page in ordered_pages: if not page.listings: break non_empty_pages.append(page) return non_empty_pages return ordered_pages def fetch_detail(self, listing_id: str | int) -> dict: try: if MOBILEDE_DETAIL_TRANSPORT == "legacy": html = self.fetch_html( self.build_legacy_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS, ) else: html = self.fetch_html( self.build_transport_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS, ) except requests.RequestException as exc: error_kind, status_code = classify_detail_error(exc) raise MobileDeDetailError( f"mobile.de detail request failed: listing_id={listing_id} status={status_code} kind={error_kind}", error_kind=error_kind, status_code=status_code, ) from exc try: detail = extract_detail_listing(html) except Exception as exc: raise MobileDeDetailParseError( f"mobile.de detail payload parse failed: listing_id={listing_id}: {exc}", error_kind="parse_error", ) from exc if not detail: raise MobileDeDetailParseError( f"mobile.de detail payload is empty or unsupported: listing_id={listing_id}", error_kind="parse_error", ) return detail @staticmethod def _map_listing(item: dict) -> MobileDeListing: listing_id = str(item.get("id") or item.get("adId") or "") attr = item.get("attr") if isinstance(item.get("attr"), dict) else {} contact = item.get("contact") if isinstance(item.get("contact"), dict) else {} location = ", ".join( part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part ) or None return MobileDeListing( id=listing_id, url=MobileDeClient.build_detail_url(listing_id) if listing_id else "", title=item.get("shortTitle"), subtitle=item.get("subTitle"), price=item.get("p"), seller_name=contact.get("name"), seller_type=contact.get("type") or item.get("st"), location=location, first_registration=attr.get("fr"), mileage=attr.get("ml"), power=attr.get("pw"), fuel=attr.get("ft"), transmission=attr.get("tr"), raw=item, )