Prepare mobile de parser release

iaai_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="IAAI Scraper API",
-        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        title="mobile.de Scraper API",
+        description="REST API для управления задачами скрапинга mobile.de и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

iaai_scraper/api/routes/health.py

@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "iaai-scraper-api",
+        "service": "mobilede-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

iaai_scraper/api/routes/tasks.py

@@ -1,14 +1,18 @@
 # Роуты запуска задач синхронизации и просмотра истории sync-runs.
 
+import json
+
 from fastapi import APIRouter, Depends, Query
 from pydantic import BaseModel
+from redis import Redis
 from sqlalchemy import select, func
 
+from ...core.config import Settings
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
-from ...worker.celery_app import IAAI_SYNC_QUEUE, celery_app
-from ...worker.tasks import sync_vehicle_task, sync_listing_task
+from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
 
 router = APIRouter()
 
@@ -26,6 +30,74 @@ class SyncListingRequest(BaseModel):
     only_new: bool | None = None
 
 
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool = True
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool = True
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
 @router.post("/tasks/sync-vehicle")
 def start_sync_vehicle(
     body: SyncVehicleRequest,
@@ -81,9 +153,40 @@ def get_task_status(task_id: str):
     elif result.info is not None:
         payload["meta"] = result.info
 
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
     return payload
 
 
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"iaai:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
 @router.get("/sync-runs")
 def list_sync_runs(
     page: int = Query(1, ge=1),

iaai_scraper/cli.py

@@ -3,11 +3,12 @@ from pathlib import Path
 
 from .core.config import Settings
 from .core.utils import save_to_json
+from .mobile_de import MobileDeClient, MobileDeScraper
 from .scraper import IAAIScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -16,21 +17,21 @@ def build_parser() -> argparse.ArgumentParser:
 
     subparsers.add_parser("init-db", help="Create DB tables")
 
-    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
     listing_parser.add_argument("--make", default=None)
     listing_parser.add_argument("--model", default=None)
     listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
 
-    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
     scrape_parser.add_argument("vehicle_url")
     scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
 
-    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
     sync_vehicle_parser.add_argument("vehicle_url")
     sync_vehicle_parser.add_argument("--lane", default="iaai")
     sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
 
-    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
     sync_listing_parser.add_argument("--make", default=None)
     sync_listing_parser.add_argument("--model", default=None)
     sync_listing_parser.add_argument("--lane", default="iaai_cars")
@@ -38,6 +39,42 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
+    mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
+    mobile_search_parser.add_argument("--page", type=int, default=1)
+    mobile_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
+    mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
+    mobile_search_parser.add_argument("--price-min", default=None)
+    mobile_search_parser.add_argument("--price-max", default=None)
+    mobile_search_parser.add_argument("--year-min", default=None)
+    mobile_search_parser.add_argument("--year-max", default=None)
+    mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
+
+    mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
+    mobile_detail_parser.add_argument("listing_id")
+    mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
+
+    mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
+    mobile_sync_search_parser.add_argument("--page", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_sync_search_parser.add_argument("--make-id", default=None)
+    mobile_sync_search_parser.add_argument("--model-id", default=None)
+    mobile_sync_search_parser.add_argument("--price-min", default=None)
+    mobile_sync_search_parser.add_argument("--price-max", default=None)
+    mobile_sync_search_parser.add_argument("--year-min", default=None)
+    mobile_sync_search_parser.add_argument("--year-max", default=None)
+    mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
+
+    mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
+    mobile_sync_detail_parser.add_argument("listing_id")
+    mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
+
     return parser
 
 
@@ -53,6 +90,55 @@ def main() -> None:
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
+    if args.command in {"search", "mobilede-search"}:
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.collect_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command in {"detail", "mobilede-detail"}:
+        scraper = MobileDeScraper()
+        data = scraper.collect_detail(args.listing_id)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-search":
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.sync_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            lane=args.lane,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-detail":
+        scraper = MobileDeScraper()
+        data = scraper.sync_detail(args.listing_id, lane=args.lane)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
     with IAAIScraper(runtime_settings) as scraper:
         if args.command == "init-db":
             data = scraper.init_db()

iaai_scraper/core/logs.py

@@ -43,3 +43,13 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
     )
     for handler in root.handlers:
         handler.setFormatter(fmt)
+
+    for logger_name in (
+        "celery.app.trace",
+        "celery.worker.request",
+        "celery.worker.strategy",
+    ):
+        noisy_logger = logging.getLogger(logger_name)
+        noisy_logger.handlers.clear()
+        noisy_logger.propagate = False
+        noisy_logger.disabled = True

iaai_scraper/core/runtime_config.py

@@ -275,11 +275,104 @@ class RuntimeFiltersConfig:
         return not self.exclude.matches(values)
 
 
+@dataclass(slots=True)
+class RuntimeMobileDeSegment:
+    make: str | None = None
+    make_id: str | None = None
+    model: str | None = None
+    model_id: str | None = None
+    search_url: str | None = None
+    only_new: bool | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    start_page: int = 1
+    max_pages: int | None = None
+    label: str | None = None
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None":
+        data = data or {}
+        make = str(data.get("make") or "").strip() or None
+        make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None
+        model = str(data.get("model") or "").strip() or None
+        model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None
+        search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None
+        if not make_id and not make and not search_url:
+            return None
+        label = str(data.get("label") or "").strip() or None
+        return cls(
+            make=make,
+            make_id=make_id,
+            model=model,
+            model_id=model_id,
+            search_url=search_url,
+            only_new=_optional_bool(data.get("only_new")),
+            price_min=str(data.get("price_min") or "").strip() or None,
+            price_max=str(data.get("price_max") or "").strip() or None,
+            year_min=str(data.get("year_min") or "").strip() or None,
+            year_max=str(data.get("year_max") or "").strip() or None,
+            start_page=max(1, _optional_int(data.get("start_page")) or 1),
+            max_pages=_optional_int(data.get("max_pages")),
+            label=label,
+        )
+
+    def to_task_kwargs(self) -> dict[str, Any]:
+        return {
+            "make": self.make,
+            "make_id": self.make_id,
+            "model": self.model,
+            "model_id": self.model_id,
+            "search_url": self.search_url,
+            "only_new": self.only_new,
+            "price_min": self.price_min,
+            "price_max": self.price_max,
+            "year_min": self.year_min,
+            "year_max": self.year_max,
+            "start_page": self.start_page,
+            "max_pages": self.max_pages,
+            "label": self.label or self.display_name,
+        }
+
+    @property
+    def display_name(self) -> str:
+        if self.label:
+            return self.label
+        if self.search_url:
+            return "filtered-url"
+        parts = [part for part in [self.make, self.model] if part]
+        return " / ".join(parts) or self.make_id or "mobilede-segment"
+
+
+@dataclass(slots=True)
+class RuntimeMobileDeConfig:
+    segments: tuple[RuntimeMobileDeSegment, ...] = ()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig":
+        data = data or {}
+        raw_segments = data.get("segments")
+        segments: list[RuntimeMobileDeSegment] = []
+        if isinstance(raw_segments, list):
+            for item in raw_segments:
+                if not isinstance(item, dict):
+                    continue
+                segment = RuntimeMobileDeSegment.from_dict(item)
+                if segment is not None:
+                    segments.append(segment)
+        return cls(segments=tuple(segments))
+
+    def is_empty(self) -> bool:
+        return not self.segments
+
+
 @dataclass(slots=True)
 class RuntimeConfig:
     sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
     listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
     filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
+    mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig)
 
     @classmethod
     def from_file(cls, config_path: str | None) -> "RuntimeConfig":
@@ -298,4 +391,5 @@ class RuntimeConfig:
             sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
             listing=RuntimeListingConfig.from_dict(payload.get("listing")),
             filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
+            mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")),
         )

iaai_scraper/mobile_de/__init__.py

@@ -0,0 +1,3 @@
+from .client import MobileDeClient
+from .scraper import MobileDeScraper
+

iaai_scraper/mobile_de/client.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections.abc import Callable, Iterable
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+import requests
+
+from .flight import extract_detail_listing, extract_search_results
+from .models import MobileDeListing, MobileDeSearchPage
+
+logger = logging.getLogger("mobile_de.client")
+
+BASE_URL = "https://www.mobile.de"
+SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
+DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
+DEFAULT_HEADERS = {
+    "user-agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "accept-language": "ru,en;q=0.9,de;q=0.8",
+}
+
+
+class MobileDeClient:
+    """HTTP client for mobile.de search/detail pages."""
+
+    def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
+        self.session = session or requests.Session()
+        self.session.headers.update(DEFAULT_HEADERS)
+        self.delay_seconds = max(0.0, delay_seconds)
+
+    @classmethod
+    def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
+        session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
+        session.mount("https://", adapter)
+        session.mount("http://", adapter)
+        return cls(session=session, delay_seconds=delay_seconds)
+
+    @staticmethod
+    def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
+        make = str(make_id).strip()
+        model = str(model_id).strip() if model_id is not None else ""
+        return f"{make};{model};;"
+
+    @staticmethod
+    def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
+        query = {
+            "sb": "rel",
+            "od": "up",
+            "vc": "Car",
+            "s": "Car",
+            "pageNumber": page_number,
+        }
+        query.update({key: value for key, value in params.items() if value is not None})
+        return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
+
+    @staticmethod
+    def build_search_url_from_existing(
+        search_url: str,
+        *,
+        page_number: int | None = None,
+        **params: str | int | None,
+    ) -> str:
+        parts = urlsplit(search_url)
+        query_items = [
+            (key, value)
+            for key, value in parse_qsl(parts.query, keep_blank_values=True)
+            if key != "pageNumber" and key not in params
+        ]
+        if page_number is not None:
+            query_items.append(("pageNumber", str(page_number)))
+        query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
+        scheme = parts.scheme or "https"
+        netloc = parts.netloc or urlsplit(BASE_URL).netloc
+        path = parts.path or SEARCH_PATH
+        return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
+
+    @staticmethod
+    def build_detail_url(listing_id: str | int) -> str:
+        query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
+        return f"{BASE_URL}{DETAIL_PATH}?{query}"
+
+    def fetch_html(self, url: str, *, timeout: int = 30) -> str:
+        response = self.session.get(url, timeout=timeout)
+        response.raise_for_status()
+        return response.text
+
+    def fetch_search_page(
+        self,
+        page_number: int = 1,
+        *,
+        search_url: str | None = None,
+        **params: str | int | None,
+    ) -> MobileDeSearchPage:
+        url = (
+            self.build_search_url_from_existing(search_url, page_number=page_number, **params)
+            if search_url
+            else self.build_search_url(page_number=page_number, **params)
+        )
+        html = self.fetch_html(url)
+        raw = extract_search_results(html)
+        listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
+        return MobileDeSearchPage(
+            url=url,
+            page_number=page_number,
+            total_results=raw.get("numResultsTotal"),
+            listings=listings,
+            raw_search_results=raw,
+        )
+
+    def iter_search_pages(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int | None = None,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> Iterable[MobileDeSearchPage]:
+        page_number = start_page
+        pages_seen = 0
+        logger.debug(
+            "mobile.de search window started: start_page=%s max_pages=%s params=%s",
+            start_page,
+            max_pages,
+            {key: value for key, value in params.items() if value is not None},
+        )
+        while max_pages is None or pages_seen < max_pages:
+            logger.debug("mobile.de fetching search page=%s", page_number)
+            page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+            page_meta = {
+                "page_number": page.page_number,
+                "pages_seen": pages_seen + 1,
+                "max_pages": max_pages,
+                "listing_count": len(page.listings),
+                "total_results": page.total_results,
+            }
+            logger.debug(
+                "mobile.de fetched search page=%s listings=%s total_results=%s",
+                page.page_number,
+                len(page.listings),
+                page.total_results,
+            )
+            if progress_callback is not None:
+                progress_callback(page, page_meta)
+            if stop_after_empty and not page.listings:
+                logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
+                break
+            yield page
+            pages_seen += 1
+            page_number += 1
+            if self.delay_seconds:
+                time.sleep(self.delay_seconds)
+        logger.debug(
+            "mobile.de search window finished: pages_seen=%s next_page=%s",
+            pages_seen,
+            page_number,
+        )
+
+    def fetch_search_pages_concurrent(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        workers: int = 8,
+        search_url: str | None = None,
+        stop_after_empty: bool = True,
+        progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
+        **params: str | int | None,
+    ) -> list[MobileDeSearchPage]:
+        max_pages = max(1, int(max_pages))
+        workers = max(1, min(int(workers), max_pages))
+        page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
+        pages_by_number: dict[int, MobileDeSearchPage] = {}
+        thread_local = threading.local()
+
+        def _fetch_page(page_number: int) -> MobileDeSearchPage:
+            client = getattr(thread_local, "client", None)
+            if client is None:
+                client = MobileDeClient.for_worker(delay_seconds=0)
+                thread_local.client = client
+            return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = {
+                executor.submit(_fetch_page, page_number): page_number
+                for page_number in page_numbers
+            }
+            for future in as_completed(futures):
+                page_number = futures[future]
+                page = future.result()
+                pages_by_number[page_number] = page
+                if progress_callback is not None:
+                    progress_callback(
+                        page,
+                        {
+                            "page_number": page.page_number,
+                            "pages_seen": len(pages_by_number),
+                            "max_pages": max_pages,
+                            "listing_count": len(page.listings),
+                            "total_results": page.total_results,
+                        },
+                    )
+        ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
+        if stop_after_empty:
+            non_empty_pages: list[MobileDeSearchPage] = []
+            for page in ordered_pages:
+                if not page.listings:
+                    break
+                non_empty_pages.append(page)
+            return non_empty_pages
+        return ordered_pages
+
+    def fetch_detail(self, listing_id: str | int) -> dict:
+        html = self.fetch_html(self.build_detail_url(listing_id))
+        return extract_detail_listing(html)
+
+    @staticmethod
+    def _map_listing(item: dict) -> MobileDeListing:
+        listing_id = str(item.get("id") or item.get("adId") or "")
+        attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
+        contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
+        location = ", ".join(
+            part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
+        ) or None
+        return MobileDeListing(
+            id=listing_id,
+            url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
+            title=item.get("shortTitle"),
+            subtitle=item.get("subTitle"),
+            price=item.get("p"),
+            seller_name=contact.get("name"),
+            seller_type=contact.get("type") or item.get("st"),
+            location=location,
+            first_registration=attr.get("fr"),
+            mileage=attr.get("ml"),
+            power=attr.get("pw"),
+            fuel=attr.get("ft"),
+            transmission=attr.get("tr"),
+            raw=item,
+        )

iaai_scraper/mobile_de/flight.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
+
+
+def extract_next_flight_strings(html: str) -> list[str]:
+    """Extract decoded Next.js Flight chunks from mobile.de HTML."""
+    chunks: list[str] = []
+    for match in NEXT_FLIGHT_RE.finditer(html):
+        raw = match.group(1)
+        try:
+            chunks.append(json.loads(f'"{raw}"'))
+        except json.JSONDecodeError:
+            # Fallback keeps parser useful if one chunk has non-standard escaping.
+            chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
+    return chunks
+
+
+def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
+    """Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
+    marker_index = text.find(marker)
+    if marker_index < 0:
+        return None
+    start = text.find("{", marker_index + len(marker))
+    if start < 0:
+        return None
+
+    depth = 0
+    in_string = False
+    escaped = False
+    for index in range(start, len(text)):
+        char = text[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif char == '"':
+                in_string = False
+            continue
+        if char == '"':
+            in_string = True
+        elif char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+            if depth == 0:
+                candidate = text[start : index + 1]
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def extract_search_results(html: str) -> dict[str, Any]:
+    """Extract searchResults from mobile.de SRP HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
+            continue
+        results = extract_json_object_after(chunk, '"searchResults":')
+        if isinstance(results, dict):
+            return results
+    return {}
+
+
+def extract_detail_listing(html: str) -> dict[str, Any]:
+    """Extract listing object from mobile.de VIP/detail HTML."""
+    for chunk in extract_next_flight_strings(html):
+        if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
+            continue
+        listing = extract_json_object_after(chunk, '"listing":')
+        if isinstance(listing, dict):
+            return listing
+    return {}

iaai_scraper/mobile_de/mapper.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+from .client import MobileDeClient
+from .models import MobileDeListing
+
+_BODY_MAP = {
+    "cabrio": "OPEN",
+    "кабриолет": "OPEN",
+    "limousine": "SEDAN",
+    "седан": "SEDAN",
+    "suv": "SUV",
+    "внедорожник": "SUV",
+    "kombi": "STATION_WAGON",
+    "универсал": "STATION_WAGON",
+    "van": "MINIVAN",
+    "фургон": "MINIVAN",
+    "coupe": "COUPE",
+    "купе": "COUPE",
+    "kleinwagen": "HATCHBACK",
+    "маленький": "HATCHBACK",
+}
+
+_GEARBOX_MAP = {
+    "автомат": "AT",
+    "automatic": "AT",
+    "механ": "MT",
+    "manual": "MT",
+    "cvt": "CVT",
+}
+
+_COLOR_MAP = {
+    "schwarz": "black",
+    "черный": "black",
+    "weiß": "white",
+    "weiss": "white",
+    "белый": "white",
+    "серый": "gray",
+    "grau": "gray",
+    "silber": "silver",
+    "сереб": "silver",
+    "rot": "red",
+    "красный": "red",
+    "blau": "blue",
+    "синий": "blue",
+    "grün": "green",
+    "gruen": "green",
+    "зеленый": "green",
+}
+
+
+class MobileDeMapper:
+    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
+
+    def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
+        raw = listing.raw or {}
+        attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
+        make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
+        model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
+
+        brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing.id))
+        title = " ".join(part for part in [listing.title, listing.subtitle] if part)
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
+            price=self._money_to_int(listing.price or raw.get("p")),
+            currency="EUR",
+            mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
+            country="NA",
+            is_sold=False,
+            color=self._normalize_color(attr.get("ecol")),
+            drive=None,
+            gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
+            steering_wheel="LEFT",
+            body_type=self._normalize_body(attr.get("c")),
+            engine_volume=self._int_from_text(attr.get("cc")),
+            selling_type="CLASSIFIED",
+            one_owner=(str(attr.get("pvo") or "").strip() == "1"),
+            new_car=False,
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=listing.url,
+            origin_id=origin_id,
+            is_damaged=bool(raw.get("hasDamage")),
+            evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
+            non_smoking=True,
+            rental=False,
+            repair_history=bool(raw.get("hasDamage")),
+            slug=self._slugify(title or f"{brand} {model}"),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(raw),
+        )
+
+    def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
+        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
+        subtitle = self._text(detail.get("subTitle"))
+        fake_listing = MobileDeListing(
+            id=str(listing_id),
+            url=MobileDeClient.build_detail_url(listing_id),
+            title=title,
+            subtitle=subtitle,
+            price=self._text(detail.get("price") or detail.get("p")),
+            raw=detail,
+        )
+        return self.listing_to_car_record(fake_listing)
+
+    @staticmethod
+    def origin_id(listing_id: str) -> str:
+        return f"mobile.de:{listing_id}"
+
+    @staticmethod
+    def _parser_id(origin_id: str) -> str:
+        digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
+        return f"mobilede-{digest}"
+
+    @staticmethod
+    def _text(value: Any) -> str:
+        return "" if value is None else str(value).strip()
+
+    @classmethod
+    def _money_to_int(cls, value: Any) -> int | None:
+        return cls._int_from_text(value)
+
+    @staticmethod
+    def _int_from_text(value: Any) -> int | None:
+        if value is None:
+            return None
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return int(value)
+        digits = re.sub(r"[^0-9]", "", str(value))
+        return int(digits) if digits else None
+
+    @staticmethod
+    def _year_from_first_registration(value: Any) -> int | None:
+        text = "" if value is None else str(value)
+        match = re.search(r"(19|20)\d{2}", text)
+        return int(match.group(0)) if match else None
+
+    @staticmethod
+    def _brand_from_title(title: str | None) -> str | None:
+        if not title:
+            return None
+        return title.split()[0]
+
+    @staticmethod
+    def _model_from_title(title: str | None, brand: str) -> str | None:
+        if not title:
+            return None
+        rest = title.replace(brand, "", 1).strip()
+        return rest or None
+
+    @staticmethod
+    def _normalize_gearbox(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _GEARBOX_MAP.items():
+            if marker in text:
+                return mapped
+        return None
+
+    @staticmethod
+    def _normalize_body(value: Any) -> str:
+        text = "" if value is None else str(value).lower()
+        for marker, mapped in _BODY_MAP.items():
+            if marker in text:
+                return mapped
+        return "OTHER"
+
+    @staticmethod
+    def _normalize_color(value: Any) -> str:
+        text = "" if value is None else str(value).lower().strip()
+        for marker, mapped in _COLOR_MAP.items():
+            if marker in text:
+                return mapped
+        return text[:50] if text else "other"
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
+        return slug[:180] or "mobilede-car"
+
+    @staticmethod
+    def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
+        urls: list[str] = []
+        image = raw.get("image")
+        if isinstance(image, str):
+            urls.append(MobileDeMapper._normalize_image_url(image))
+        images = raw.get("images")
+        if isinstance(images, list):
+            for item in images:
+                if isinstance(item, str):
+                    urls.append(MobileDeMapper._normalize_image_url(item))
+                elif isinstance(item, dict):
+                    src = item.get("src") or item.get("url") or item.get("uri")
+                    if src:
+                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
+        return [
+            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
+            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+        ]
+
+    @staticmethod
+    def _normalize_image_url(value: str) -> str:
+        url = str(value).strip()
+        if not url:
+            return ""
+        if url.startswith("//"):
+            return f"https:{url}"
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+        return f"https://{url.lstrip('/')}"

iaai_scraper/mobile_de/models.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass(frozen=True)
+class MobileDeListing:
+    """One listing extracted from mobile.de search results."""
+
+    id: str
+    url: str
+    title: str | None = None
+    subtitle: str | None = None
+    price: str | None = None
+    seller_name: str | None = None
+    seller_type: str | None = None
+    location: str | None = None
+    first_registration: str | None = None
+    mileage: str | None = None
+    power: str | None = None
+    fuel: str | None = None
+    transmission: str | None = None
+    raw: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MobileDeSearchPage:
+    """Parsed mobile.de search page."""
+
+    url: str
+    page_number: int
+    total_results: int | None
+    listings: list[MobileDeListing]
+    raw_search_results: dict[str, Any] = field(default_factory=dict)

iaai_scraper/mobile_de/scraper.py

@@ -0,0 +1,325 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Callable
+from dataclasses import asdict
+from typing import Any
+
+from ..core.config import Settings, settings
+from ..storage.db import PersistenceService
+from .client import MobileDeClient
+from .mapper import MobileDeMapper
+from .models import MobileDeListing
+
+logger = logging.getLogger("mobile_de.scraper")
+
+MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
+MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
+
+
+class MobileDeScraper:
+    """High-level mobile.de scraper facade."""
+
+    def __init__(
+        self,
+        client: MobileDeClient | None = None,
+        persistence: PersistenceService | None = None,
+        mapper: MobileDeMapper | None = None,
+        runtime_settings: Settings | None = None,
+    ) -> None:
+        self.settings = runtime_settings or settings
+        self.client = client or MobileDeClient()
+        self.persistence = persistence or PersistenceService(self.settings)
+        self.mapper = mapper or MobileDeMapper()
+
+    def collect_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        params: dict[str, str | None] = {}
+        if not search_url:
+            params = {
+                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+            }
+            if make_id:
+                params["ms"] = self.client.build_make_model_param(make_id, model_id)
+        if sort_by:
+            params["sb"] = sort_by
+        if sort_order:
+            params["od"] = sort_order
+
+        logger.debug(
+            "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
+            start_page,
+            max_pages,
+            bool(search_url),
+            make_id,
+            model_id,
+            year_min,
+            year_max,
+            price_min,
+            price_max,
+            mileage_min,
+            mileage_max,
+        )
+        pages = []
+
+        def _on_page(page, meta: dict[str, int | None]) -> None:
+            payload = {
+                **meta,
+                "page_url": page.url,
+                "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
+                + len({listing.id for listing in page.listings if listing.id}),
+            }
+            if progress_callback is not None:
+                progress_callback("page_collected", payload)
+
+        concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
+        if concurrent_pages > 1 and max_pages and max_pages > 1:
+            pages.extend(self.client.fetch_search_pages_concurrent(
+                start_page=start_page,
+                max_pages=max_pages,
+                workers=concurrent_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ))
+        else:
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages.append(page)
+
+        unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
+        result = {
+            "source": "mobile.de",
+            "strategy_note": (
+                "mobile.de UI shows 50 pages, but pageNumber works beyond 50; "
+                "for full coverage split by make/model/year/price and deduplicate by id."
+            ),
+            "search_url": search_url,
+            "pages": [asdict(page) for page in pages],
+            "listing_count": sum(len(page.listings) for page in pages),
+            "unique_listing_count": len(unique_ids),
+            "unique_listing_ids": unique_ids,
+        }
+        logger.debug(
+            "mobile.de collect_search finished: pages=%s listings=%s unique=%s",
+            len(pages),
+            result["listing_count"],
+            result["unique_listing_count"],
+        )
+        if progress_callback is not None:
+            progress_callback(
+                "search_collection_done",
+                {
+                    "pages_collected": len(pages),
+                    "listing_count": result["listing_count"],
+                    "unique_listing_count": result["unique_listing_count"],
+                },
+            )
+        return result
+
+    def collect_detail(self, listing_id: str) -> dict[str, Any]:
+        return {
+            "source": "mobile.de",
+            "listing_id": str(listing_id),
+            "url": self.client.build_detail_url(listing_id),
+            "listing": self.client.fetch_detail(listing_id),
+        }
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "source": "mobile.de"}
+
+    def sync_search(
+        self,
+        *,
+        start_page: int = 1,
+        max_pages: int = 1,
+        lane: str = "mobile_de_cars",
+        only_new: bool | None = None,
+        search_url: str | None = None,
+        make_id: str | None = None,
+        model_id: str | None = None,
+        price_min: str | None = None,
+        price_max: str | None = None,
+        year_min: str | None = None,
+        year_max: str | None = None,
+        mileage_min: str | None = None,
+        mileage_max: str | None = None,
+        sort_by: str | None = None,
+        sort_order: str | None = None,
+        progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+    ) -> dict[str, Any]:
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+        failed = 0
+        logger.debug(
+            "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
+            run_id,
+            lane,
+            start_page,
+            max_pages,
+        )
+        try:
+            data = self.collect_search(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                make_id=make_id,
+                model_id=model_id,
+                price_min=price_min,
+                price_max=price_max,
+                year_min=year_min,
+                year_max=year_max,
+                mileage_min=mileage_min,
+                mileage_max=mileage_max,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                progress_callback=progress_callback,
+            )
+            # Map serialized listings back to records.
+            records = []
+            for page in data["pages"]:
+                for item in page["listings"]:
+                    records.append(self.mapper.listing_to_car_record(MobileDeListing(**item)))
+
+            skipped_existing = 0
+            if only_new and records:
+                existing_origin_ids = self.persistence.get_existing_origin_ids(
+                    [record.origin_id for record in records if record.origin_id]
+                )
+                before_filter_count = len(records)
+
+                # For newest-first, cut tail after existing streak.
+                should_cut_tail = (
+                    str(sort_by or "").lower() == "doc"
+                    and str(sort_order or "").lower() == "down"
+                    and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
+                )
+                if should_cut_tail:
+                    filtered_records = []
+                    existing_streak = 0
+                    considered = 0
+                    for record in records:
+                        considered += 1
+                        is_existing = record.origin_id in existing_origin_ids
+                        if is_existing:
+                            skipped_existing += 1
+                            existing_streak += 1
+                            if (
+                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
+                                and len(filtered_records) >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
+                            ):
+                                break
+                            continue
+                        existing_streak = 0
+                        filtered_records.append(record)
+                    records = filtered_records
+                    data["listing_count"] = considered
+                    data["unique_listing_count"] = min(int(data.get("unique_listing_count", considered)), considered)
+                    logger.info(
+                        "mobile.de only_new head-cut applied: considered=%s kept=%s skipped_existing=%s streak=%s",
+                        considered,
+                        len(records),
+                        skipped_existing,
+                        MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
+                    )
+                else:
+                    records = [record for record in records if record.origin_id not in existing_origin_ids]
+                    skipped_existing = before_filter_count - len(records)
+                logger.info(
+                    "mobile.de only_new filter applied: kept=%s skipped_existing=%s",
+                    len(records),
+                    skipped_existing,
+                )
+
+            if progress_callback is not None:
+                progress_callback(
+                    "records_mapped",
+                    {
+                        "record_count": len(records),
+                        "skipped_existing": skipped_existing,
+                        "only_new": bool(only_new),
+                        "run_id": run_id,
+                    },
+                )
+
+            upsert = self.persistence.upsert_cars_batch(records) if records else {
+                "inserted": 0,
+                "updated": 0,
+                "images_upserted": 0,
+            }
+            logger.debug(
+                "mobile.de sync_search upsert finished: run_id=%s inserted=%s updated=%s images=%s",
+                run_id,
+                upsert.get("inserted", 0),
+                upsert.get("updated", 0),
+                upsert.get("images_upserted", 0),
+            )
+            if progress_callback is not None:
+                progress_callback(
+                    "db_upsert_done",
+                    {
+                        "run_id": run_id,
+                        "inserted": int(upsert.get("inserted", 0)),
+                        "updated": int(upsert.get("updated", 0)),
+                        "images_upserted": int(upsert.get("images_upserted", 0)),
+                    },
+                )
+            self.persistence.finish_sync_run(
+                run_id,
+                status="success",
+                ids_fetched=len(records),
+                cars_upserted=int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)),
+                cars_failed=failed,
+                images_upserted=int(upsert.get("images_upserted", 0)),
+            )
+            logger.debug(
+                "mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
+                run_id,
+                data.get("listing_count", 0),
+                data.get("unique_listing_count", 0),
+            )
+            return {"run_id": run_id, "source": "mobile.de", "upsert": upsert, "skipped_existing": skipped_existing, **data}
+        except Exception as exc:
+            self.persistence.finish_sync_run(
+                run_id,
+                status="failed",
+                ids_fetched=0,
+                cars_upserted=0,
+                cars_failed=failed,
+                images_upserted=0,
+                error_summary=str(exc),
+            )
+            logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
+            raise
+
+    def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
+        self.persistence.create_tables()
+        detail = self.client.fetch_detail(listing_id)
+        record = self.mapper.detail_to_car_record(str(listing_id), detail)
+        result = self.persistence.upsert_car(record)
+        return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}

iaai_scraper/storage/enums.py

@@ -19,6 +19,7 @@ BODY_TYPE_ENUM_VALUES = (
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
 	"IAAI",
+	"MOBILE_DE",
 	"NA",
 )
-SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA")

iaai_scraper/worker/celery_app.py

@@ -15,6 +15,7 @@ from ..core.logs import setup_logging
 logger = logging.getLogger("iaai_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 IAAI_SYNC_QUEUE = "iaai_sync"
+MOBILEDE_SYNC_QUEUE = "mobilede_sync"
 PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
 
 
@@ -108,18 +109,25 @@ celery_app.conf.update(
     worker_redirect_stdouts=False,
     worker_hijack_root_logger=False,
     beat_schedule={
-        "periodic-sync-listing": {
-            "task": "iaai.sync_cars_feed",
+        "periodic-mobilede-sync-search": {
+            "task": "mobilede.sync_runtime_segments",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "kwargs": {
+                "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+                "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+                "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+            },
             "options": {
-                "queue": IAAI_SYNC_QUEUE,
+                "queue": MOBILEDE_SYNC_QUEUE,
                 "expires": settings.celery.beat_sync_interval_minutes * 60.0,
             },
         }
     },
     task_routes={
+        "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
+        "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
         "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
         "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
     },
@@ -186,10 +194,14 @@ def _on_worker_ready(**kwargs):
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
         return
 
-    logger.info("Worker ready — dispatching initial sync_listing task")
+    logger.info("Worker ready — dispatching initial mobile.de sync_search task")
     celery_app.send_task(
-        "iaai.sync_cars_feed",
-        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
-        queue=IAAI_SYNC_QUEUE,
+        "mobilede.sync_runtime_segments",
+        kwargs={
+            "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
+            "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
+            "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
         expires=settings.celery.beat_sync_interval_minutes * 60.0,
     )