from __future__ import annotations import logging import os from collections.abc import Callable from dataclasses import asdict from typing import Any from ..core.config import Settings, settings from ..storage.db import PersistenceService from ..storage.schemas import CarRecord from .client import MobileDeClient from .mapper import MobileDeMapper from .models import MobileDeListing logger = logging.getLogger("mobile_de.scraper") MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2"))) MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1"))) MOBILEDE_SEARCH_STRATEGY_NOTE = ( "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; " "for full coverage split into narrower segments and deduplicate by id." ) class MobileDeScraper: """High-level mobile.de scraper facade.""" def __init__( self, client: MobileDeClient | None = None, persistence: PersistenceService | None = None, mapper: MobileDeMapper | None = None, runtime_settings: Settings | None = None, ) -> None: self.settings = runtime_settings or settings self.client = client or MobileDeClient() self.persistence = persistence or PersistenceService(self.settings) self.mapper = mapper or MobileDeMapper() @staticmethod def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool: return ( str(sort_by or "").lower() == "doc" and str(sort_order or "").lower() == "down" and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0 ) def _build_search_params( self, *, search_url: str | None, make_id: str | None, model_id: str | None, price_min: str | None, price_max: str | None, year_min: str | None, year_max: str | None, mileage_min: str | None, mileage_max: str | None, sort_by: str | None, sort_order: str | None, ) -> dict[str, str | None]: params: dict[str, str | None] = {} if not search_url: params = { "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, } if make_id: params["ms"] = self.client.build_make_model_param(make_id, model_id) if sort_by: params["sb"] = sort_by if sort_order: params["od"] = sort_order return params @staticmethod def _dedupe_page_records( page_records: list[CarRecord], seen_record_keys: set[str], ) -> list[CarRecord]: deduped_page_records: list[CarRecord] = [] for record in page_records: record_key = record.origin_id or record.origin_url if not record_key or record_key in seen_record_keys: continue seen_record_keys.add(record_key) deduped_page_records.append(record) return deduped_page_records def _apply_only_new_page_policy( self, *, page_records: list[CarRecord], only_new: bool | None, sort_by: str | None, sort_order: str | None, skipped_existing: int, existing_streak: int, new_records_kept: int, ) -> tuple[list[CarRecord], int, int, int, bool]: if not only_new or not page_records: return page_records, skipped_existing, existing_streak, new_records_kept, False existing_origin_ids = self.persistence.get_existing_origin_ids( [record.origin_id for record in page_records if record.origin_id] ) head_cut_triggered = False should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) if should_cut_tail: filtered_records: list[CarRecord] = [] for record_index, record in enumerate(page_records): is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) if is_existing: existing_streak += 1 if ( existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS ): head_cut_triggered = True skipped_existing += max(0, len(page_records) - record_index - 1) break else: existing_streak = 0 new_records_kept += 1 filtered_records.append(record) return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered for record in page_records: if not record.origin_id or record.origin_id not in existing_origin_ids: new_records_kept += 1 return page_records, skipped_existing, existing_streak, new_records_kept, False def collect_search( self, *, start_page: int = 1, max_pages: int = 1, search_url: str | None = None, make_id: str | None = None, model_id: str | None = None, price_min: str | None = None, price_max: str | None = None, year_min: str | None = None, year_max: str | None = None, mileage_min: str | None = None, mileage_max: str | None = None, sort_by: str | None = None, sort_order: str | None = None, progress_callback: Callable[[str, dict[str, Any]], None] | None = None, ) -> dict[str, Any]: params = self._build_search_params( search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, sort_by=sort_by, sort_order=sort_order, ) logger.debug( "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", start_page, max_pages, bool(search_url), make_id, model_id, year_min, year_max, price_min, price_max, mileage_min, mileage_max, ) pages = [] def _on_page(page, meta: dict[str, int | None]) -> None: payload = { **meta, "page_url": page.url, "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id}) + len({listing.id for listing in page.listings if listing.id}), } if progress_callback is not None: progress_callback("page_collected", payload) concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1"))) if concurrent_pages > 1 and max_pages and max_pages > 1: pages.extend(self.client.fetch_search_pages_concurrent( start_page=start_page, max_pages=max_pages, workers=concurrent_pages, search_url=search_url, progress_callback=_on_page, **params, )) else: for page in self.client.iter_search_pages( start_page=start_page, max_pages=max_pages, search_url=search_url, progress_callback=_on_page, **params, ): pages.append(page) unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id}) result = { "source": "mobile.de", "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE, "search_url": search_url, "pages": [asdict(page) for page in pages], "listing_count": sum(len(page.listings) for page in pages), "unique_listing_count": len(unique_ids), "unique_listing_ids": unique_ids, } logger.debug( "mobile.de collect_search finished: pages=%s listings=%s unique=%s", len(pages), result["listing_count"], result["unique_listing_count"], ) if progress_callback is not None: progress_callback( "search_collection_done", { "pages_collected": len(pages), "listing_count": result["listing_count"], "unique_listing_count": result["unique_listing_count"], }, ) return result def collect_detail(self, listing_id: str) -> dict[str, Any]: return { "source": "mobile.de", "listing_id": str(listing_id), "url": self.client.build_detail_url(listing_id), "listing": self.client.fetch_detail(listing_id), } def init_db(self) -> dict[str, Any]: self.persistence.create_tables() return {"status": "ok", "source": "mobile.de"} def sync_search( self, *, start_page: int = 1, max_pages: int = 1, lane: str = "mobile_de_cars", only_new: bool | None = None, search_url: str | None = None, make_id: str | None = None, model_id: str | None = None, price_min: str | None = None, price_max: str | None = None, year_min: str | None = None, year_max: str | None = None, mileage_min: str | None = None, mileage_max: str | None = None, sort_by: str | None = None, sort_order: str | None = None, progress_callback: Callable[[str, dict[str, Any]], None] | None = None, ) -> dict[str, Any]: self.persistence.create_tables() run_id = self.persistence.start_sync_run(lane) pages_payload: list[dict[str, Any]] = [] unique_ids: set[str] = set() listing_count = 0 skipped_existing = 0 ids_fetched = 0 cars_upserted = 0 inserted_total = 0 updated_total = 0 images_upserted = 0 existing_streak = 0 new_records_kept = 0 head_cut_triggered = False seen_record_keys: set[str] = set() logger.debug( "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s", run_id, lane, start_page, max_pages, ) try: data = self.collect_search( start_page=start_page, max_pages=max_pages, search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, sort_by=sort_by, sort_order=sort_order, progress_callback=progress_callback, ) pages_payload = list(data.get("pages", [])) listing_count = int(data.get("listing_count", 0) or 0) unique_ids = set(data.get("unique_listing_ids", [])) pages_collected = len(pages_payload) records: list[CarRecord] = [] for page in pages_payload: page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])] records.extend(self._dedupe_page_records(page_records, seen_record_keys)) if only_new and records: existing_origin_ids = self.persistence.get_existing_origin_ids( [record.origin_id for record in records if record.origin_id] ) should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) if should_cut_tail: filtered_records: list[CarRecord] = [] for record_index, record in enumerate(records): is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) filtered_records.append(record) if is_existing: existing_streak += 1 if ( existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS ): head_cut_triggered = True skipped_existing += max(0, len(records) - record_index - 1) break else: existing_streak = 0 new_records_kept += 1 records = filtered_records if progress_callback is not None: progress_callback( "records_mapped", { "record_count": len(records), "skipped_existing": skipped_existing, "only_new": bool(only_new), "run_id": run_id, "pages_collected": pages_collected, }, ) upsert = self.persistence.upsert_cars_batch(records) if records else { "inserted": 0, "updated": 0, "images_upserted": 0, } ids_fetched = len(records) inserted_total = int(upsert.get("inserted", 0)) updated_total = int(upsert.get("updated", 0)) images_upserted = int(upsert.get("images_upserted", 0)) cars_upserted = inserted_total + updated_total logger.debug( "mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s", run_id, pages_collected, inserted_total, updated_total, images_upserted, ) if progress_callback is not None: progress_callback( "db_upsert_done", { "run_id": run_id, "pages_collected": pages_collected, "pages_in_batch": pages_collected, "inserted": inserted_total, "updated": updated_total, "images_upserted": images_upserted, }, ) if head_cut_triggered: logger.info( "mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s", len(records), skipped_existing, MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK, ) data["early_stopped"] = head_cut_triggered self.persistence.finish_sync_run( run_id, status="success", ids_fetched=ids_fetched, cars_upserted=cars_upserted, cars_failed=0, images_upserted=images_upserted, ) logger.debug( "mobile.de sync_search completed: run_id=%s listings=%s unique=%s", run_id, data.get("listing_count", 0), data.get("unique_listing_count", 0), ) return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data} except Exception as exc: self.persistence.finish_sync_run( run_id, status="failed", ids_fetched=ids_fetched, cars_upserted=cars_upserted, cars_failed=0, images_upserted=images_upserted, error_summary=str(exc), ) logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True) raise def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]: self.persistence.create_tables() detail = self.client.fetch_detail(listing_id) record = self.mapper.detail_to_car_record(str(listing_id), detail) result = self.persistence.upsert_car(record) return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}