add mobilede scraper

mobilede_scraper/mobile_de/__init__.py

@@ -0,0 +1,3 @@
+from .client import MobileDeClient
+from .scraper import MobileDeScraper
+

mobilede_scraper/mobile_de/client.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+import logging
+import os
+import random
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections.abc import Callable, Iterable
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import requests
+
+from ..core.config import ProxyConfig
+from .flight import extract_detail_listing, extract_search_results
+from .models import MobileDeListing, MobileDeSearchPage
+
+logger = logging.getLogger("mobile_de.client")
+
+BASE_URL = "https://www.mobile.de"
+SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DEFAULT_HEADERS = {
+    "user-agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "accept-language": "ru,en;q=0.9,de;q=0.8",
+}
+MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
+MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
+MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
+MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
+MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+
+
+class MobileDeClient:
+    """HTTP client for mobile.de search/detail pages."""
+
+    def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
+        self.session = session or requests.Session()
+        self.session.headers.update(DEFAULT_HEADERS)
+        self.delay_seconds = max(0.0, delay_seconds)
+
+    @staticmethod
+    def _is_retryable_status(status_code: int) -> bool:
+        return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
+
+    @staticmethod
+    def _compute_backoff(attempt: int) -> float:
+        base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
+        bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base
+        jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0
+        return max(0.0, bounded + jitter)
+
+    @classmethod
+    def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
+        session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        proxy_cfg = ProxyConfig()
+        proxies = proxy_cfg.to_requests_proxies()
+        if proxies:
+            session.proxies.update(proxies)
+            logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server)
+        return cls(session=session, delay_seconds=delay_seconds)
+
+    @staticmethod
+    def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
+        make = str(make_id).strip()
+        model = str(model_id).strip() if model_id is not None else ""
+        return f"{make};{model};;"
+
+    @staticmethod
+    def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
+        query = {
+            "sb": "rel",
+            "od": "up",
+            "vc": "Car",
+            "s": "Car",
+            "pageNumber": page_number,
+        }
+        query.update({key: value for key, value in params.items() if value is not None})
+        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+
+    @staticmethod
+    def build_search_url_from_existing(
+        search_url: str,
+        *,
+        page_number: int | None = None,
+        **params: str | int | None,
+    ) -> str:
+        parts = urlsplit(search_url)
+        query_items = [
+            (key, value)
+            for key, value in parse_qsl(parts.query, keep_blank_values=True)
+            if key != "pageNumber" and key not in params
+        ]
+        if page_number is not None:
+            query_items.append(("pageNumber", str(page_number)))
+        query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
+        scheme = parts.scheme or "https"
+        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        path = parts.path or SEARCH_PATH
+        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
+
+    @staticmethod
+    def build_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+
+    def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+        last_error: Exception | None = None
+        attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1)
+        for attempt in range(1, attempts + 1):
+            try:
+                response = self.session.get(url, timeout=timeout)
+                if self._is_retryable_status(response.status_code) and attempt < attempts:
+                    sleep_seconds = self._compute_backoff(attempt)
+                    logger.warning(
+                        "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
+                        response.status_code,
+                        attempt,
+                        attempts,
+                        sleep_seconds,
+                        url,
+                    )
+                    if sleep_seconds:
+                        time.sleep(sleep_seconds)
+                    continue
+                response.raise_for_status()
+                return response.text
+            except requests.RequestException as exc:
+                last_error = exc
+                status_code = getattr(getattr(exc, "response", None), "status_code", None)
+                retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
+                if attempt >= attempts or not retryable:
+                    raise
+                sleep_seconds = self._compute_backoff(attempt)
+                logger.warning(
+                    "mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s",
+                    attempt,
+                    attempts,
+                    status_code,
+                    sleep_seconds,
+                    url,
+                    exc,
+                )
+                if sleep_seconds:
+                    time.sleep(sleep_seconds)
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("mobile.de fetch_html failed without a captured exception")
+
+    def fetch_search_page(
+        self,
+        page_number: int = 1,
+        *,
+        search_url: str | None = None,
+        **params: str | int | None,
+    ) -> MobileDeSearchPage:
+        url = (
+            self.build_search_url_from_existing(search_url, page_number=page_number, **params)
+            if search_url
+            else self.build_search_url(page_number=page_number, **params)
+        )
+        html = self.fetch_html(url)
+        raw = extract_search_results(html)
+        listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
+        return MobileDeSearchPage(
+            url=url,
+            page_number=page_number,
+            total_results=raw.get("numResultsTotal"),
+            listings=listings,
+            raw_search_results=raw,
+        )
+
+    def iter_search_pages(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int | None = None,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> Iterable[MobileDeSearchPage]:
+        page_number = start_page
+        pages_seen = 0
+        logger.debug(
+            "mobile.de search window started: start_page=%s max_pages=%s params=%s",
+            start_page,
+            max_pages,
+            {key: value for key, value in params.items() if value is not None},
+        )
+        while max_pages is None or pages_seen < max_pages:
+            logger.debug("mobile.de fetching search page=%s", page_number)
+            page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+            page_meta = {
+                "page_number": page.page_number,
+                "pages_seen": pages_seen + 1,
+                "max_pages": max_pages,
+                "listing_count": len(page.listings),
+                "total_results": page.total_results,
+            }
+            logger.debug(
+                "mobile.de fetched search page=%s listings=%s total_results=%s",
+                page.page_number,
+                len(page.listings),
+                page.total_results,
+            )
+            if progress_callback is not None:
+                progress_callback(page, page_meta)
+            if stop_after_empty and not page.listings:
+                logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
+                break
+            yield page
+            pages_seen += 1
+            page_number += 1
+            if self.delay_seconds:
+                time.sleep(self.delay_seconds)
+        logger.debug(
+            "mobile.de search window finished: pages_seen=%s next_page=%s",
+            pages_seen,
+            page_number,
+        )
+
+    def fetch_search_pages_concurrent(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        workers: int = 8,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> list[MobileDeSearchPage]:
+        max_pages = max(1, int(max_pages))
+        workers = max(1, min(int(workers), max_pages))
+        page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
+        pages_by_number: dict[int, MobileDeSearchPage] = {}
+        thread_local = threading.local()
+
+        def _fetch_page(page_number: int) -> MobileDeSearchPage:
+            client = getattr(thread_local, "client", None)
+            if client is None:
+                client = MobileDeClient.for_worker(delay_seconds=0)
+                thread_local.client = client
+            return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = {
+                executor.submit(_fetch_page, page_number): page_number
+                for page_number in page_numbers
+            }
+            for future in as_completed(futures):
+                page_number = futures[future]
+                page = future.result()
+                pages_by_number[page_number] = page
+                if progress_callback is not None:
+                    progress_callback(
+                        page,
+                        {
+                            "page_number": page.page_number,
+                            "pages_seen": len(pages_by_number),
+                            "max_pages": max_pages,
+                            "listing_count": len(page.listings),
+                            "total_results": page.total_results,
+                        },
+                    )
+        ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
+        if stop_after_empty:
+            non_empty_pages: list[MobileDeSearchPage] = []
+            for page in ordered_pages:
+                if not page.listings:
+                    break
+                non_empty_pages.append(page)
+            return non_empty_pages
+        return ordered_pages
+
+    def fetch_detail(self, listing_id: str | int) -> dict:
+        html = self.fetch_html(self.build_detail_url(listing_id))
+        return extract_detail_listing(html)
+
+    @staticmethod
+    def _map_listing(item: dict) -> MobileDeListing:
+        listing_id = str(item.get("id") or item.get("adId") or "")
+        attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
+        contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
+        location = ", ".join(
+            part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
+        ) or None
+        return MobileDeListing(
+            id=listing_id,
+            url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
+            title=item.get("shortTitle"),
+            subtitle=item.get("subTitle"),
+            price=item.get("p"),
+            seller_name=contact.get("name"),
+            seller_type=contact.get("type") or item.get("st"),
+            location=location,
+            first_registration=attr.get("fr"),
+            mileage=attr.get("ml"),
+            power=attr.get("pw"),
+            fuel=attr.get("ft"),
+            transmission=attr.get("tr"),
+            raw=item,
+        )

mobilede_scraper/mobile_de/flight.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+
+
+def extract_next_flight_strings(html: str) -> list[str]:
+    """Extract decoded Next.js Flight chunks from mobile.de HTML."""
+    chunks: list[str] = []
+    for match in NEXT_FLIGHT_RE.finditer(html):
+        raw = match.group(1)
+        try:
+            chunks.append(json.loads(f'"{raw}"'))
+        except json.JSONDecodeError:
+            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
+    return chunks
+
+
+def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
+    """Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
+    marker_index = text.find(marker)
+    if marker_index < 0:
+        return None
+    start = text.find("{", marker_index + len(marker))
+    if start < 0:
+        return None
+
+    depth = 0
+    in_string = False
+    escaped = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : index + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def extract_search_results(html: str) -> dict[str, Any]:
+    """Extract searchResults from mobile.de SRP HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
+            continue
+        results = extract_json_object_after(chunk, '"searchResults":')
+        if isinstance(results, dict):
+            return results
+    return {}
+
+
+def extract_detail_listing(html: str) -> dict[str, Any]:
+    """Extract listing object from mobile.de VIP/detail HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
+            continue
+        listing = extract_json_object_after(chunk, '"listing":')
+        if isinstance(listing, dict):
+            return listing
+    return {}

mobilede_scraper/mobile_de/mapper.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+from .client import MobileDeClient
+from .models import MobileDeListing
+
+_BODY_MAP = {
+    "cabrio": "OPEN",
+    "кабриолет": "OPEN",
+    "limousine": "SEDAN",
+    "седан": "SEDAN",
+    "suv": "SUV",
+    "внедорожник": "SUV",
+    "kombi": "STATION_WAGON",
+    "универсал": "STATION_WAGON",
+    "van": "MINIVAN",
+    "фургон": "MINIVAN",
+    "coupe": "COUPE",
+    "купе": "COUPE",
+    "kleinwagen": "HATCHBACK",
+    "маленький": "HATCHBACK",
+}
+
+_GEARBOX_MAP = {
+    "автомат": "AT",
+    "automatic": "AT",
+    "механ": "MT",
+    "manual": "MT",
+    "cvt": "CVT",
+}
+
+_COLOR_MAP = {
+    "schwarz": "black",
+    "черный": "black",
+    "weiß": "white",
+    "weiss": "white",
+    "белый": "white",
+    "серый": "gray",
+    "grau": "gray",
+    "silber": "silver",
+    "сереб": "silver",
+    "rot": "red",
+    "красный": "red",
+    "blau": "blue",
+    "синий": "blue",
+    "grün": "green",
+    "gruen": "green",
+    "зеленый": "green",
+}
+
+
+class MobileDeMapper:
+    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
+
+    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
+        raw = listing.raw or {}
+        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
+        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
+        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
+
+        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing.id))
+        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
+            price=self._money_to_int(listing.price or raw.get("p")),
+            currency="EUR",
+            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
+            country="NA",
+            is_sold=False,
+            color=self._normalize_color(attr.get("ecol")),
+            drive=None,
+            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
+            steering_wheel="LEFT",
+            body_type=self._normalize_body(attr.get("c")),
+            engine_volume=self._int_from_text(attr.get("cc")),
+            selling_type="CLASSIFIED",
+            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
+            new_car=False,
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=listing.url,
+            origin_id=origin_id,
+            is_damaged=bool(raw.get("hasDamage")),
+            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
+            non_smoking=True,
+            rental=False,
+            repair_history=bool(raw.get("hasDamage")),
+            slug=self._slugify(title or f"{brand} {model}"),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(raw),
+        )
+
+    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
+        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
+        subtitle = self._text(detail.get("subTitle"))
+        fake_listing = MobileDeListing(
+            id=str(listing_id),
+            url=MobileDeClient.build_detail_url(listing_id),
+            title=title,
+            subtitle=subtitle,
+            price=self._text(detail.get("price") or detail.get("p")),
+            raw=detail,
+        )
+        return self.listing_to_car_record(fake_listing)
+
+    @staticmethod
+    def origin_id(listing_id: str) -> str:
+        return f"mobile.de:{listing_id}"
+
+    @staticmethod
+    def _parser_id(origin_id: str) -> str:
+        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
+        return f"mobilede-{digest}"
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return "" if value is None else str(value).strip()
+
+    @classmethod
+    def _money_to_int(cls, value: Any) -> int | None:
+        return cls._int_from_text(value)
+
+    @staticmethod
+    def _int_from_text(value: Any) -> int | None:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return int(value)
+        digits = re.sub(r"[^0-9]", "", str(value))
+        return int(digits) if digits else None
+
+    @staticmethod
+    def _year_from_first_registration(value: Any) -> int | None:
+        text = "" if value is None else str(value)
+        match = re.search(r"(19|20)\d{2}", text)
+        return int(match.group(0)) if match else None
+
+    @staticmethod
+    def _brand_from_title(title: str | None) -> str | None:
+        if not title:
+            return None
+        return title.split()[0]
+
+    @staticmethod
+    def _model_from_title(title: str | None, brand: str) -> str | None:
+        if not title:
+            return None
+        rest = title.replace(brand, "", 1).strip()
+        return rest or None
+
+    @staticmethod
+    def _normalize_gearbox(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_body(value: Any) -> str:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _BODY_MAP.items():
+            if marker in text:
+                return mapped
+        return "OTHER"
+
+    @staticmethod
+    def _normalize_color(value: Any) -> str:
+        text = "" if value is None else str(value).lower().strip()
+        for marker, mapped in _COLOR_MAP.items():
+            if marker in text:
+                return mapped
+        return text[:50] if text else "other"
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
+        return slug[:180] or "mobilede-car"
+
+    @staticmethod
+    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
+        urls: list[str] = []
+        image = raw.get("image")
+        if isinstance(image, str):
+            urls.append(MobileDeMapper._normalize_image_url(image))
+        images = raw.get("images")
+        if isinstance(images, list):
+            for item in images:
+                if isinstance(item, str):
+                    urls.append(MobileDeMapper._normalize_image_url(item))
+                elif isinstance(item, dict):
+                    src = item.get("src") or item.get("url") or item.get("uri")
+                    if src:
+                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
+        return [
+            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+        ]
+
+    @staticmethod
+    def _normalize_image_url(value: str) -> str:
+        url = str(value).strip()
+        if not url:
+            return ""
+        if url.startswith("//"):
+            return f"https:{url}"
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+        return f"https://{url.lstrip('/')}"

mobilede_scraper/mobile_de/models.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class MobileDeListing:
+    """One listing extracted from mobile.de search results."""
+
+    id: str
+    url: str
+    title: str | None = None
+    subtitle: str | None = None
+    price: str | None = None
+    seller_name: str | None = None
+    seller_type: str | None = None
+    location: str | None = None
+    first_registration: str | None = None
+    mileage: str | None = None
+    power: str | None = None
+    fuel: str | None = None
+    transmission: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MobileDeSearchPage:
+    """Parsed mobile.de search page."""
+
+    url: str
+    page_number: int
+    total_results: int | None
+    listings: list[MobileDeListing]
+    raw_search_results: dict[str, Any] = field(default_factory=dict)

mobilede_scraper/mobile_de/scraper.py

@@ -0,0 +1,454 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any
+
+import requests
+
+from ..core.config import Settings, settings
+from ..storage.db import PersistenceService
+from ..storage.schemas import CarRecord
+from .client import MobileDeClient
+from .mapper import MobileDeMapper
+from .models import MobileDeListing
+
+logger = logging.getLogger("mobile_de.scraper")
+
+MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
+MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+MOBILEDE_SEARCH_STRATEGY_NOTE = (
+    "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
+    "for full coverage split into narrower segments and deduplicate by id."
+)
+
+
+class MobileDeScraper:
+    """High-level mobile.de scraper facade."""
+
+    def __init__(
+        self,
+        client: MobileDeClient | None = None,
+        persistence: PersistenceService | None = None,
+        mapper: MobileDeMapper | None = None,
+        runtime_settings: Settings | None = None,
+    ) -> None:
+        self.settings = runtime_settings or settings
+        self.client = client or MobileDeClient()
+        self.persistence = persistence or PersistenceService(self.settings)
+        self.mapper = mapper or MobileDeMapper()
+
+    @staticmethod
+    def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
+        return (
+            str(sort_by or "").lower() == "doc"
+            and str(sort_order or "").lower() == "down"
+            and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
+        )
+
+    def _build_search_params(
+        self,
+        *,
+        search_url: str | None,
+        make_id: str | None,
+        model_id: str | None,
+        price_min: str | None,
+        price_max: str | None,
+        year_min: str | None,
+        year_max: str | None,
+        mileage_min: str | None,
+        mileage_max: str | None,
+        sort_by: str | None,
+        sort_order: str | None,
+    ) -> dict[str, str | None]:
+        params: dict[str, str | None] = {}
+        if not search_url:
+            params = {
+                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+            }
+            if make_id:
+                params["ms"] = self.client.build_make_model_param(make_id, model_id)
+        if sort_by:
+            params["sb"] = sort_by
+        if sort_order:
+            params["od"] = sort_order
+        return params
+
+    @staticmethod
+    def _dedupe_page_records(
+        page_records: list[CarRecord],
+        seen_record_keys: set[str],
+    ) -> list[CarRecord]:
+        deduped_page_records: list[CarRecord] = []
+        for record in page_records:
+            record_key = record.origin_id or record.origin_url
+            if not record_key or record_key in seen_record_keys:
+                continue
+            seen_record_keys.add(record_key)
+            deduped_page_records.append(record)
+        return deduped_page_records
+
+    def _apply_only_new_page_policy(
+        self,
+        *,
+        page_records: list[CarRecord],
+        only_new: bool | None,
+        sort_by: str | None,
+        sort_order: str | None,
+        skipped_existing: int,
+        existing_streak: int,
+        new_records_kept: int,
+    ) -> tuple[list[CarRecord], int, int, int, bool]:
+        if not only_new or not page_records:
+            return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+        existing_origin_ids = self.persistence.get_existing_origin_ids(
+            [record.origin_id for record in page_records if record.origin_id]
+        )
+        head_cut_triggered = False
+        should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
+
+        if should_cut_tail:
+            filtered_records: list[CarRecord] = []
+            for record_index, record in enumerate(page_records):
+                is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
+                if is_existing:
+                    existing_streak += 1
+                    if (
+                        existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                        and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                    ):
+                        head_cut_triggered = True
+                        skipped_existing += max(0, len(page_records) - record_index - 1)
+                        break
+                else:
+                    existing_streak = 0
+                    new_records_kept += 1
+                filtered_records.append(record)
+            return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
+
+        for record in page_records:
+            if not record.origin_id or record.origin_id not in existing_origin_ids:
+                new_records_kept += 1
+        return page_records, skipped_existing, existing_streak, new_records_kept, False
+
+    def collect_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        params = self._build_search_params(
+            search_url=search_url,
+            make_id=make_id,
+            model_id=model_id,
+            price_min=price_min,
+            price_max=price_max,
+            year_min=year_min,
+            year_max=year_max,
+            mileage_min=mileage_min,
+            mileage_max=mileage_max,
+            sort_by=sort_by,
+            sort_order=sort_order,
+        )
+
+        logger.debug(
+            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
+            start_page,
+            max_pages,
+            bool(search_url),
+            make_id,
+            model_id,
+            year_min,
+            year_max,
+            price_min,
+            price_max,
+            mileage_min,
+            mileage_max,
+        )
+        pages = []
+
+        def _on_page(page, meta: dict[str, int | None]) -> None:
+            payload = {
+                **meta,
+                "page_url": page.url,
+                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
+                + len({listing.id for listing in page.listings if listing.id}),
+            }
+            if progress_callback is not None:
+                progress_callback("page_collected", payload)
+
+        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+        if concurrent_pages > 1 and max_pages and max_pages > 1:
+            try:
+                pages.extend(self.client.fetch_search_pages_concurrent(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    workers=concurrent_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ))
+            except requests.RequestException as exc:
+                logger.warning(
+                    "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
+                    concurrent_pages,
+                    start_page,
+                    max_pages,
+                    exc,
+                )
+                pages.clear()
+                for page in self.client.iter_search_pages(
+                    start_page=start_page,
+                    max_pages=max_pages,
+                    search_url=search_url,
+                    progress_callback=_on_page,
+                    **params,
+                ):
+                    pages.append(page)
+        else:
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages.append(page)
+
+        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
+        result = {
+            "source": "mobile.de",
+            "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
+            "search_url": search_url,
+            "pages": [asdict(page) for page in pages],
+            "listing_count": sum(len(page.listings) for page in pages),
+            "unique_listing_count": len(unique_ids),
+            "unique_listing_ids": unique_ids,
+        }
+        logger.debug(
+            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
+            len(pages),
+            result["listing_count"],
+            result["unique_listing_count"],
+        )
+        if progress_callback is not None:
+            progress_callback(
+                "search_collection_done",
+                {
+                    "pages_collected": len(pages),
+                    "listing_count": result["listing_count"],
+                    "unique_listing_count": result["unique_listing_count"],
+                },
+            )
+        return result
+
+    def collect_detail(self, listing_id: str) -> dict[str, Any]:
+        return {
+            "source": "mobile.de",
+            "listing_id": str(listing_id),
+            "url": self.client.build_detail_url(listing_id),
+            "listing": self.client.fetch_detail(listing_id),
+        }
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "source": "mobile.de"}
+
+    def sync_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        lane: str = "mobile_de_cars",
+        only_new: bool | None = None,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+        pages_payload: list[dict[str, Any]] = []
+        unique_ids: set[str] = set()
+        listing_count = 0
+        skipped_existing = 0
+        ids_fetched = 0
+        cars_upserted = 0
+        inserted_total = 0
+        updated_total = 0
+        images_upserted = 0
+        existing_streak = 0
+        new_records_kept = 0
+        head_cut_triggered = False
+        seen_record_keys: set[str] = set()
+        logger.debug(
+            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
+            run_id,
+            lane,
+            start_page,
+            max_pages,
+        )
+        try:
+            data = self.collect_search(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                make_id=make_id,
+                model_id=model_id,
+                price_min=price_min,
+                price_max=price_max,
+                year_min=year_min,
+                year_max=year_max,
+                mileage_min=mileage_min,
+                mileage_max=mileage_max,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                progress_callback=progress_callback,
+            )
+
+            pages_payload = list(data.get("pages", []))
+            listing_count = int(data.get("listing_count", 0) or 0)
+            unique_ids = set(data.get("unique_listing_ids", []))
+            pages_collected = len(pages_payload)
+
+            records: list[CarRecord] = []
+            for page in pages_payload:
+                page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
+                records.extend(self._dedupe_page_records(page_records, seen_record_keys))
+
+            if only_new and records:
+                existing_origin_ids = self.persistence.get_existing_origin_ids(
+                    [record.origin_id for record in records if record.origin_id]
+                )
+                should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
+                if should_cut_tail:
+                    filtered_records: list[CarRecord] = []
+                    for record_index, record in enumerate(records):
+                        is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
+                        filtered_records.append(record)
+                        if is_existing:
+                            existing_streak += 1
+                            if (
+                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                                and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                            ):
+                                head_cut_triggered = True
+                                skipped_existing += max(0, len(records) - record_index - 1)
+                                break
+                        else:
+                            existing_streak = 0
+                            new_records_kept += 1
+                    records = filtered_records
+
+            if progress_callback is not None:
+                progress_callback(
+                    "records_mapped",
+                    {
+                        "record_count": len(records),
+                        "skipped_existing": skipped_existing,
+                        "only_new": bool(only_new),
+                        "run_id": run_id,
+                        "pages_collected": pages_collected,
+                    },
+                )
+
+            upsert = self.persistence.upsert_cars_batch(records) if records else {
+                "inserted": 0,
+                "updated": 0,
+                "images_upserted": 0,
+            }
+            ids_fetched = len(records)
+            inserted_total = int(upsert.get("inserted", 0))
+            updated_total = int(upsert.get("updated", 0))
+            images_upserted = int(upsert.get("images_upserted", 0))
+            cars_upserted = inserted_total + updated_total
+
+            logger.debug(
+                "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
+                run_id,
+                pages_collected,
+                inserted_total,
+                updated_total,
+                images_upserted,
+            )
+            if progress_callback is not None:
+                progress_callback(
+                    "db_upsert_done",
+                    {
+                        "run_id": run_id,
+                        "pages_collected": pages_collected,
+                        "pages_in_batch": pages_collected,
+                        "inserted": inserted_total,
+                        "updated": updated_total,
+                        "images_upserted": images_upserted,
+                    },
+                )
+
+            if head_cut_triggered:
+                logger.info(
+                    "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
+                    len(records),
+                    skipped_existing,
+                    MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
+                )
+
+            data["early_stopped"] = head_cut_triggered
+            self.persistence.finish_sync_run(
+                run_id,
+                status="success",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+            )
+            logger.debug(
+                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
+                run_id,
+                data.get("listing_count", 0),
+                data.get("unique_listing_count", 0),
+            )
+            return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
+        except Exception as exc:
+            self.persistence.finish_sync_run(
+                run_id,
+                status="failed",
+                ids_fetched=ids_fetched,
+                cars_upserted=cars_upserted,
+                cars_failed=0,
+                images_upserted=images_upserted,
+                error_summary=str(exc),
+            )
+            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            raise
+
+    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        record = self.mapper.detail_to_car_record(str(listing_id), detail)
+        result = self.persistence.upsert_car(record)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}