refactor mobile.de parser, fix country mapping, update README

mobilede_scraper/mobile_de/client.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import os
 import random
-import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from collections.abc import Callable, Iterable
@@ -34,6 +33,16 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC
 MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
 MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
 MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
+MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
+MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
+MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
+MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
+MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
+MOBILEDE_FLARESOLVERR_STATUSES = {
+    int(item.strip())
+    for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
+    if item.strip().isdigit()
+}
 
 
 class MobileDeClient:
@@ -43,11 +52,24 @@ class MobileDeClient:
         self.session = session or requests.Session()
         self.session.headers.update(DEFAULT_HEADERS)
         self.delay_seconds = max(0.0, delay_seconds)
+        self.proxy_config = ProxyConfig()
+        proxies = self.proxy_config.to_requests_proxies()
+        if proxies:
+            self.session.proxies.update(proxies)
 
     @staticmethod
     def _is_retryable_status(status_code: int) -> bool:
         return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
 
+    @staticmethod
+    def _should_use_flaresolverr(status_code: int | None) -> bool:
+        return (
+            MOBILEDE_FLARESOLVERR_ENABLED
+            and bool(MOBILEDE_FLARESOLVERR_URL)
+            and status_code is not None
+            and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
+        )
+
     @staticmethod
     def _compute_backoff(attempt: int) -> float:
         base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
@@ -61,12 +83,10 @@ class MobileDeClient:
         adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
         session.mount("https://", adapter)
         session.mount("http://", adapter)
-        proxy_cfg = ProxyConfig()
-        proxies = proxy_cfg.to_requests_proxies()
-        if proxies:
-            session.proxies.update(proxies)
-            logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server)
-        return cls(session=session, delay_seconds=delay_seconds)
+        client = cls(session=session, delay_seconds=delay_seconds)
+        if client.proxy_config.enabled:
+            logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
+        return client
 
     @staticmethod
     def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
@@ -118,19 +138,25 @@ class MobileDeClient:
         for attempt in range(1, attempts + 1):
             try:
                 response = self.session.get(url, timeout=timeout)
-                if self._is_retryable_status(response.status_code) and attempt < attempts:
-                    sleep_seconds = self._compute_backoff(attempt)
-                    logger.warning(
-                        "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
-                        response.status_code,
-                        attempt,
-                        attempts,
-                        sleep_seconds,
-                        url,
-                    )
-                    if sleep_seconds:
-                        time.sleep(sleep_seconds)
-                    continue
+                if self._is_retryable_status(response.status_code):
+                    if attempt < attempts:
+                        sleep_seconds = self._compute_backoff(attempt)
+                        logger.warning(
+                            "mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
+                            response.status_code,
+                            attempt,
+                            attempts,
+                            sleep_seconds,
+                            url,
+                        )
+                        if sleep_seconds:
+                            time.sleep(sleep_seconds)
+                        continue
+                    if self._should_use_flaresolverr(response.status_code):
+                        try:
+                            return self._fetch_html_with_flaresolverr(url)
+                        except Exception as exc:
+                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
                 response.raise_for_status()
                 return response.text
             except requests.RequestException as exc:
@@ -138,6 +164,11 @@ class MobileDeClient:
                 status_code = getattr(getattr(exc, "response", None), "status_code", None)
                 retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
                 if attempt >= attempts or not retryable:
+                    if self._should_use_flaresolverr(status_code):
+                        try:
+                            return self._fetch_html_with_flaresolverr(url)
+                        except Exception as flaresolverr_exc:
+                            logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
                     raise
                 sleep_seconds = self._compute_backoff(attempt)
                 logger.warning(
@@ -155,6 +186,56 @@ class MobileDeClient:
             raise last_error
         raise RuntimeError("mobile.de fetch_html failed without a captured exception")
 
+    def _fetch_html_with_flaresolverr(self, url: str) -> str:
+        payload: dict[str, object] = {
+            "cmd": "request.get",
+            "url": url,
+            "maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
+        }
+        if MOBILEDE_FLARESOLVERR_SESSION:
+            payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
+
+        response = requests.post(
+            MOBILEDE_FLARESOLVERR_URL,
+            json=payload,
+            timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
+        )
+        response.raise_for_status()
+        data = response.json()
+        if data.get("status") != "ok":
+            raise RuntimeError(str(data.get("message") or data))
+
+        solution = data.get("solution")
+        if not isinstance(solution, dict):
+            raise RuntimeError("FlareSolverr response does not contain solution")
+
+        html = solution.get("response")
+        if not isinstance(html, str) or not html:
+            raise RuntimeError("FlareSolverr response does not contain HTML")
+
+        user_agent = solution.get("userAgent")
+        if isinstance(user_agent, str) and user_agent:
+            self.session.headers.update({"user-agent": user_agent})
+
+        cookies = solution.get("cookies")
+        if isinstance(cookies, list):
+            for cookie in cookies:
+                if not isinstance(cookie, dict):
+                    continue
+                name = cookie.get("name")
+                value = cookie.get("value")
+                if not isinstance(name, str) or not isinstance(value, str):
+                    continue
+                self.session.cookies.set(
+                    name,
+                    value,
+                    domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
+                    path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
+                )
+
+        logger.info("mobile.de fetched via FlareSolverr url=%s", url)
+        return html
+
     def fetch_search_page(
         self,
         page_number: int = 1,

mobilede_scraper/mobile_de/mapper.py

@@ -1,4 +1,4 @@
-from __future__ import annotations
+from __future__ import annotations
 
 import hashlib
 import re
@@ -11,23 +11,27 @@ from .models import MobileDeListing
 
 _BODY_MAP = {
     "cabrio": "OPEN",
-    "кабриолет": "OPEN",
+    "roadster": "OPEN",
+    "кабрио": "OPEN",
     "limousine": "SEDAN",
-    "седан": "SEDAN",
+    "sedan": "SEDAN",
+    "сeдан": "SEDAN",
     "suv": "SUV",
-    "внедорожник": "SUV",
+    "внедорож": "SUV",
     "kombi": "STATION_WAGON",
+    "estate": "STATION_WAGON",
     "универсал": "STATION_WAGON",
     "van": "MINIVAN",
-    "фургон": "MINIVAN",
+    "минивэн": "MINIVAN",
     "coupe": "COUPE",
     "купе": "COUPE",
+    "hatchback": "HATCHBACK",
     "kleinwagen": "HATCHBACK",
-    "маленький": "HATCHBACK",
 }
 
 _GEARBOX_MAP = {
     "автомат": "AT",
+    "automatik": "AT",
     "automatic": "AT",
     "механ": "MT",
     "manual": "MT",
@@ -36,26 +40,33 @@ _GEARBOX_MAP = {
 
 _COLOR_MAP = {
     "schwarz": "black",
-    "черный": "black",
-    "weiß": "white",
+    "черн": "black",
+    "black": "black",
     "weiss": "white",
-    "белый": "white",
-    "серый": "gray",
+    "weiß": "white",
+    "бел": "white",
+    "white": "white",
     "grau": "gray",
+    "сер": "gray",
+    "gray": "gray",
     "silber": "silver",
     "сереб": "silver",
+    "silver": "silver",
     "rot": "red",
-    "красный": "red",
+    "красн": "red",
+    "red": "red",
     "blau": "blue",
-    "синий": "blue",
+    "син": "blue",
+    "blue": "blue",
     "grün": "green",
     "gruen": "green",
-    "зеленый": "green",
+    "зелен": "green",
+    "green": "green",
 }
 
 
 class MobileDeMapper:
-    """Map mobile.de search/detail payloads into the existing CarRecord schema."""
+    """Map mobile.de payloads into CarRecord."""
 
     def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
         raw = listing.raw or {}
@@ -76,7 +87,7 @@ class MobileDeMapper:
             price=self._money_to_int(listing.price or raw.get("p")),
             currency="EUR",
             mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
-            country="NA",
+            country=self._normalize_country("DE"),
             is_sold=False,
             color=self._normalize_color(attr.get("ecol")),
             drive=None,
@@ -102,17 +113,64 @@ class MobileDeMapper:
         )
 
     def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
-        title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
+        attrs = self._detail_attrs_by_tag(detail.get("attributes"))
+        make = detail.get("make") if isinstance(detail.get("make"), dict) else {}
+        model_payload = detail.get("model") if isinstance(detail.get("model"), dict) else {}
+        contact = detail.get("contact") if isinstance(detail.get("contact"), dict) else {}
+
+        short_title = self._text(detail.get("shortTitle") or make.get("localized") or "UNKNOWN")
         subtitle = self._text(detail.get("subTitle"))
-        fake_listing = MobileDeListing(
-            id=str(listing_id),
-            url=MobileDeClient.build_detail_url(listing_id),
-            title=title,
-            subtitle=subtitle,
-            price=self._text(detail.get("price") or detail.get("p")),
-            raw=detail,
+        title = " ".join(part for part in [short_title, subtitle] if part)
+
+        brand = self._text(make.get("localized") or self._brand_from_title(short_title) or "UNKNOWN")
+        model = self._text(model_payload.get("localized") or self._model_from_title(short_title, brand) or subtitle or "UNKNOWN")
+        origin_id = self.origin_id(str(listing_id))
+
+        price_amount, price_currency = self._detail_price(detail.get("price"))
+        mileage = self._int_from_text(attrs.get("mileage")) or 0
+        year = self._year_from_first_registration(attrs.get("firstRegistration"))
+        gearbox = self._normalize_gearbox(attrs.get("transmission"))
+        body_type = self._normalize_body(attrs.get("category") or detail.get("category"))
+        color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName"))
+
+        damage_text = self._text(attrs.get("damageCondition")).lower()
+        is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
+
+        owners_text = self._text(attrs.get("numPreviousOwners"))
+        one_owner = owners_text in {"1", "01", "1.0"}
+
+        return CarRecord(
+            parser_id=self._parser_id(origin_id),
+            brand=brand[:50] or "UNKNOWN",
+            model=model[:50] or "UNKNOWN",
+            year=year,
+            price=price_amount,
+            currency=price_currency or "EUR",
+            mileage=mileage,
+            country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
+            is_sold=False,
+            color=color,
+            drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")),
+            gearbox=gearbox,
+            steering_wheel="LEFT",
+            body_type=body_type,
+            engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")),
+            selling_type="CLASSIFIED",
+            one_owner=one_owner,
+            new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
+            is_hidden=False,
+            origin="MOBILE_DE",
+            origin_url=MobileDeClient.build_detail_url(listing_id),
+            origin_id=origin_id,
+            is_damaged=is_damaged,
+            evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None,
+            non_smoking=True,
+            rental=False,
+            repair_history=is_damaged,
+            slug=self._slugify(title or f"{brand} {model}"),
+            last_seen_at=datetime.now(timezone.utc),
+            images=self._images_from_listing(detail),
         )
-        return self.listing_to_car_record(fake_listing)
 
     @staticmethod
     def origin_id(listing_id: str) -> str:
@@ -129,6 +187,9 @@ class MobileDeMapper:
 
     @classmethod
     def _money_to_int(cls, value: Any) -> int | None:
+        if isinstance(value, dict):
+            amount = ((value.get("grs") or {}).get("amount") if isinstance(value.get("grs"), dict) else None) or value.get("amount")
+            return cls._int_from_text(amount)
         return cls._int_from_text(value)
 
     @staticmethod
@@ -167,6 +228,32 @@ class MobileDeMapper:
                 return mapped
         return None
 
+    @staticmethod
+    def _normalize_drive(value: Any) -> str | None:
+        text = "" if value is None else str(value).lower()
+        if any(marker in text for marker in ("front", "fwd", "перед")):
+            return "FWD"
+        if any(marker in text for marker in ("rear", "rwd", "зад")):
+            return "RWD"
+        if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")):
+            return "4WD"
+        return None
+
+    @staticmethod
+    def _normalize_country(value: Any) -> str:
+        text = "" if value is None else str(value).strip().upper()
+        if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
+            return "DE"
+        if text in {"US", "USA", "UNITED STATES"}:
+            return "US"
+        if text in {"CA", "CANADA"}:
+            return "CA"
+        if text in {"JP", "JAPAN"}:
+            return "JP"
+        if text in {"KR", "KOREA", "SOUTH KOREA"}:
+            return "KR"
+        return "NA"
+
     @staticmethod
     def _normalize_body(value: Any) -> str:
         text = "" if value is None else str(value).lower()
@@ -188,6 +275,33 @@ class MobileDeMapper:
         slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
         return slug[:180] or "mobilede-car"
 
+    @staticmethod
+    def _detail_attrs_by_tag(value: Any) -> dict[str, str]:
+        result: dict[str, str] = {}
+        if not isinstance(value, list):
+            return result
+        for item in value:
+            if not isinstance(item, dict):
+                continue
+            tag = str(item.get("tag") or "").strip()
+            val = str(item.get("value") or "").strip()
+            if tag and val and tag not in result:
+                result[tag] = val
+        return result
+
+    @staticmethod
+    def _detail_price(value: Any) -> tuple[int | None, str | None]:
+        if not isinstance(value, dict):
+            return None, None
+        grs = value.get("grs") if isinstance(value.get("grs"), dict) else {}
+        amount = grs.get("amount") if isinstance(grs, dict) else None
+        currency = grs.get("currency") if isinstance(grs, dict) else None
+        if amount is None:
+            amount = value.get("amount")
+        if currency is None:
+            currency = value.get("currency")
+        return MobileDeMapper._int_from_text(amount), (str(currency).strip() if currency else None)
+
     @staticmethod
     def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
         urls: list[str] = []
@@ -203,6 +317,13 @@ class MobileDeMapper:
                     src = item.get("src") or item.get("url") or item.get("uri")
                     if src:
                         urls.append(MobileDeMapper._normalize_image_url(str(src)))
+        media_gallery = raw.get("mediaGallery")
+        if isinstance(media_gallery, list):
+            for item in media_gallery:
+                if isinstance(item, dict):
+                    src = item.get("uri") or item.get("url")
+                    if src:
+                        urls.append(MobileDeMapper._normalize_image_url(str(src)))
         return [
             ImageRecord(fullres_image=url, preview_image=url, order_index=index)
             for index, url in enumerate(dict.fromkeys(url for url in urls if url))

mobilede_scraper/mobile_de/scraper.py

@@ -314,9 +314,7 @@ class MobileDeScraper:
             max_pages,
         )
         try:
-            data = self.collect_search(
-                start_page=start_page,
-                max_pages=max_pages,
+            params = self._build_search_params(
                 search_url=search_url,
                 make_id=make_id,
                 model_id=model_id,
@@ -328,96 +326,121 @@ class MobileDeScraper:
                 mileage_max=mileage_max,
                 sort_by=sort_by,
                 sort_order=sort_order,
-                progress_callback=progress_callback,
             )
 
-            pages_payload = list(data.get("pages", []))
-            listing_count = int(data.get("listing_count", 0) or 0)
-            unique_ids = set(data.get("unique_listing_ids", []))
-            pages_collected = len(pages_payload)
+            def _on_page(page, meta: dict[str, int | None]) -> None:
+                if progress_callback is None:
+                    return
+                payload = {
+                    **meta,
+                    "page_url": page.url,
+                    "unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
+                }
+                progress_callback("page_collected", payload)
 
-            records: list[CarRecord] = []
-            for page in pages_payload:
-                page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
-                records.extend(self._dedupe_page_records(page_records, seen_record_keys))
+            pages_collected = 0
+            early_stopped = False
+            for page in self.client.iter_search_pages(
+                start_page=start_page,
+                max_pages=max_pages,
+                search_url=search_url,
+                progress_callback=_on_page,
+                **params,
+            ):
+                pages_collected += 1
+                pages_payload.append(asdict(page))
+                listing_count += len(page.listings)
+                unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
 
-            if only_new and records:
-                existing_origin_ids = self.persistence.get_existing_origin_ids(
-                    [record.origin_id for record in records if record.origin_id]
-                )
-                should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
-                if should_cut_tail:
-                    filtered_records: list[CarRecord] = []
-                    for record_index, record in enumerate(records):
-                        is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
-                        filtered_records.append(record)
-                        if is_existing:
-                            existing_streak += 1
-                            if (
-                                existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
-                                and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
-                            ):
-                                head_cut_triggered = True
-                                skipped_existing += max(0, len(records) - record_index - 1)
-                                break
-                        else:
-                            existing_streak = 0
-                            new_records_kept += 1
-                    records = filtered_records
-
-            if progress_callback is not None:
-                progress_callback(
-                    "records_mapped",
-                    {
-                        "record_count": len(records),
-                        "skipped_existing": skipped_existing,
-                        "only_new": bool(only_new),
-                        "run_id": run_id,
-                        "pages_collected": pages_collected,
-                    },
+                page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
+                page_records = self._dedupe_page_records(page_records, seen_record_keys)
+                page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
+                    self._apply_only_new_page_policy(
+                        page_records=page_records,
+                        only_new=only_new,
+                        sort_by=sort_by,
+                        sort_order=sort_order,
+                        skipped_existing=skipped_existing,
+                        existing_streak=existing_streak,
+                        new_records_kept=new_records_kept,
+                    )
                 )
 
-            upsert = self.persistence.upsert_cars_batch(records) if records else {
-                "inserted": 0,
-                "updated": 0,
-                "images_upserted": 0,
+                if progress_callback is not None:
+                    progress_callback(
+                        "records_mapped",
+                        {
+                            "record_count": len(page_records),
+                            "skipped_existing": skipped_existing,
+                            "only_new": bool(only_new),
+                            "run_id": run_id,
+                            "pages_collected": pages_collected,
+                            "page_number": page.page_number,
+                        },
+                    )
+
+                upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
+                    "inserted": 0,
+                    "updated": 0,
+                    "images_upserted": 0,
+                }
+                page_inserted = int(upsert.get("inserted", 0))
+                page_updated = int(upsert.get("updated", 0))
+                page_images = int(upsert.get("images_upserted", 0))
+
+                ids_fetched += len(page_records)
+                inserted_total += page_inserted
+                updated_total += page_updated
+                images_upserted += page_images
+                cars_upserted = inserted_total + updated_total
+
+                logger.debug(
+                    "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
+                    run_id,
+                    page.page_number,
+                    page_inserted,
+                    page_updated,
+                    page_images,
+                )
+                if progress_callback is not None:
+                    progress_callback(
+                        "db_upsert_done",
+                        {
+                            "run_id": run_id,
+                            "pages_collected": pages_collected,
+                            "pages_in_batch": 1,
+                            "page_number": page.page_number,
+                            "inserted": page_inserted,
+                            "updated": page_updated,
+                            "images_upserted": page_images,
+                        },
+                    )
+
+                if head_cut_triggered:
+                    early_stopped = True
+                    break
+
+            data = {
+                "source": "mobile.de",
+                "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
+                "search_url": search_url,
+                "pages": pages_payload,
+                "listing_count": listing_count,
+                "unique_listing_count": len(unique_ids),
+                "unique_listing_ids": sorted(unique_ids),
+                "early_stopped": early_stopped,
             }
-            ids_fetched = len(records)
-            inserted_total = int(upsert.get("inserted", 0))
-            updated_total = int(upsert.get("updated", 0))
-            images_upserted = int(upsert.get("images_upserted", 0))
-            cars_upserted = inserted_total + updated_total
 
-            logger.debug(
-                "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
-                run_id,
-                pages_collected,
-                inserted_total,
-                updated_total,
-                images_upserted,
-            )
             if progress_callback is not None:
                 progress_callback(
-                    "db_upsert_done",
+                    "search_collection_done",
                     {
-                        "run_id": run_id,
                         "pages_collected": pages_collected,
-                        "pages_in_batch": pages_collected,
-                        "inserted": inserted_total,
-                        "updated": updated_total,
-                        "images_upserted": images_upserted,
+                        "listing_count": listing_count,
+                        "unique_listing_count": len(unique_ids),
                     },
                 )
 
-            if head_cut_triggered:
-                logger.info(
-                    "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
-                    len(records),
-                    skipped_existing,
-                    MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
-                )
-
-            data["early_stopped"] = head_cut_triggered
             self.persistence.finish_sync_run(
                 run_id,
                 status="success",
@@ -432,7 +455,16 @@ class MobileDeScraper:
                 data.get("listing_count", 0),
                 data.get("unique_listing_count", 0),
             )
-            return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
+            return {
+                "run_id": run_id,
+                "upsert": {
+                    "inserted": inserted_total,
+                    "updated": updated_total,
+                    "images_upserted": images_upserted,
+                },
+                "skipped_existing": skipped_existing,
+                **data,
+            }
         except Exception as exc:
             self.persistence.finish_sync_run(
                 run_id,