from __future__ import annotations import logging import os from collections.abc import Callable from dataclasses import asdict from datetime import datetime, timezone from typing import Any import requests from ..core.config import Settings, settings from ..core.runtime_config import RuntimeFiltersConfig from ..storage.db import PersistenceService from ..storage.schemas import CarRecord from .client import MobileDeClient from .mapper import MobileDeMapper from .models import MobileDeListing logger = logging.getLogger("mobile_de.scraper") MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2"))) MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1"))) MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"} MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20"))) MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2"))) MOBILEDE_SEARCH_STRATEGY_NOTE = ( "mobile.de search pages return about 20 listings per page and are limited to about 50 pages; " "for full coverage split into narrower segments and deduplicate by id." ) class MobileDeScraper: """High-level mobile.de scraper facade.""" def __init__( self, client: MobileDeClient | None = None, persistence: PersistenceService | None = None, mapper: MobileDeMapper | None = None, runtime_settings: Settings | None = None, ) -> None: self.settings = runtime_settings or settings self.client = client or MobileDeClient() self.persistence = persistence or PersistenceService(self.settings) self.mapper = mapper or MobileDeMapper() @staticmethod def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool: return ( str(sort_by or "").lower() == "doc" and str(sort_order or "").lower() == "down" and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0 ) def _build_search_params( self, *, search_url: str | None, make_id: str | None, model_id: str | None, price_min: str | None, price_max: str | None, year_min: str | None, year_max: str | None, mileage_min: str | None, mileage_max: str | None, sort_by: str | None, sort_order: str | None, ) -> dict[str, str | None]: params: dict[str, str | None] = { "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, } if not search_url: if make_id: params["ms"] = self.client.build_make_model_param(make_id, model_id) if sort_by: params["sb"] = sort_by if sort_order: params["od"] = sort_order return params @staticmethod def _dedupe_page_records( page_records: list[CarRecord], seen_record_keys: set[str], ) -> list[CarRecord]: deduped_page_records: list[CarRecord] = [] for record in page_records: record_key = record.origin_id or record.origin_url if not record_key or record_key in seen_record_keys: continue seen_record_keys.add(record_key) deduped_page_records.append(record) return deduped_page_records @staticmethod def _record_key(record: CarRecord) -> str: return record.origin_id or record.origin_url @staticmethod def _record_needs_detail_enrich(record: CarRecord) -> bool: return any( ( record.year is None, record.mileage == 0, record.engine_volume is None, record.body_type == "OTHER", record.color == "other", ) ) @staticmethod def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]: return ( int(record.year is None), int(record.engine_volume is None), int(record.mileage == 0), int(record.body_type == "OTHER"), int(record.color == "other"), ) def _apply_only_new_page_policy( self, *, page_records: list[CarRecord], only_new: bool | None, sort_by: str | None, sort_order: str | None, skipped_existing: int, existing_streak: int, new_records_kept: int, existing_origin_ids: set[str] | None = None, ) -> tuple[list[CarRecord], int, int, int, bool]: if not only_new or not page_records: return page_records, skipped_existing, existing_streak, new_records_kept, False if existing_origin_ids is None: existing_origin_ids = self.persistence.get_existing_origin_ids( [record.origin_id for record in page_records if record.origin_id] ) head_cut_triggered = False should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order) if should_cut_tail: filtered_records: list[CarRecord] = [] for record_index, record in enumerate(page_records): is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids) if is_existing: existing_streak += 1 if ( existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS ): head_cut_triggered = True skipped_existing += max(0, len(page_records) - record_index - 1) break else: existing_streak = 0 new_records_kept += 1 filtered_records.append(record) return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered for record in page_records: if not record.origin_id or record.origin_id not in existing_origin_ids: new_records_kept += 1 return page_records, skipped_existing, existing_streak, new_records_kept, False def collect_search( self, *, start_page: int = 1, max_pages: int = 1, search_url: str | None = None, make_id: str | None = None, model_id: str | None = None, price_min: str | None = None, price_max: str | None = None, year_min: str | None = None, year_max: str | None = None, mileage_min: str | None = None, mileage_max: str | None = None, sort_by: str | None = None, sort_order: str | None = None, progress_callback: Callable[[str, dict[str, Any]], None] | None = None, ) -> dict[str, Any]: params = self._build_search_params( search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, sort_by=sort_by, sort_order=sort_order, ) logger.debug( "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", start_page, max_pages, bool(search_url), make_id, model_id, year_min, year_max, price_min, price_max, mileage_min, mileage_max, ) pages = [] def _on_page(page, meta: dict[str, int | None]) -> None: payload = { **meta, "page_url": page.url, "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id}) + len({listing.id for listing in page.listings if listing.id}), } if progress_callback is not None: progress_callback("page_collected", payload) concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1"))) if concurrent_pages > 1 and max_pages and max_pages > 1: try: pages.extend(self.client.fetch_search_pages_concurrent( start_page=start_page, max_pages=max_pages, workers=concurrent_pages, search_url=search_url, progress_callback=_on_page, **params, )) except requests.RequestException as exc: logger.warning( "mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s", concurrent_pages, start_page, max_pages, exc, ) pages.clear() for page in self.client.iter_search_pages( start_page=start_page, max_pages=max_pages, search_url=search_url, progress_callback=_on_page, **params, ): pages.append(page) else: for page in self.client.iter_search_pages( start_page=start_page, max_pages=max_pages, search_url=search_url, progress_callback=_on_page, **params, ): pages.append(page) unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id}) result = { "source": "mobile.de", "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE, "search_url": search_url, "pages": [asdict(page) for page in pages], "listing_count": sum(len(page.listings) for page in pages), "unique_listing_count": len(unique_ids), "unique_listing_ids": unique_ids, } logger.debug( "mobile.de collect_search finished: pages=%s listings=%s unique=%s", len(pages), result["listing_count"], result["unique_listing_count"], ) if progress_callback is not None: progress_callback( "search_collection_done", { "pages_collected": len(pages), "listing_count": result["listing_count"], "unique_listing_count": result["unique_listing_count"], }, ) return result def collect_detail(self, listing_id: str) -> dict[str, Any]: return { "source": "mobile.de", "listing_id": str(listing_id), "url": self.client.build_detail_url(listing_id), "listing": self.client.fetch_detail(listing_id), } def init_db(self) -> dict[str, Any]: self.persistence.create_tables() return {"status": "ok", "source": "mobile.de"} def sync_search( self, *, start_page: int = 1, max_pages: int = 1, lane: str = "mobile_de_cars", only_new: bool | None = None, search_url: str | None = None, make_id: str | None = None, model_id: str | None = None, price_min: str | None = None, price_max: str | None = None, year_min: str | None = None, year_max: str | None = None, mileage_min: str | None = None, mileage_max: str | None = None, sort_by: str | None = None, sort_order: str | None = None, seen_at: datetime | None = None, progress_callback: Callable[[str, dict[str, Any]], None] | None = None, runtime_filters: RuntimeFiltersConfig | None = None, limit: int | None = None, ) -> dict[str, Any]: if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None: raise ValueError( "mobile.de does not support the runtime filter flags.run_and_drive: " "the source payload has no reliable vehicle-condition field" ) self.persistence.create_tables() run_id = self.persistence.start_sync_run(lane) run_seen_at = seen_at or datetime.now(timezone.utc) if run_seen_at.tzinfo is None: run_seen_at = run_seen_at.replace(tzinfo=timezone.utc) pages_payload: list[dict[str, Any]] = [] unique_ids: set[str] = set() listing_count = 0 skipped_existing = 0 ids_fetched = 0 cars_upserted = 0 inserted_total = 0 updated_total = 0 images_upserted = 0 detail_enriched = 0 detail_enrich_failed = 0 existing_streak = 0 new_records_kept = 0 head_cut_triggered = False seen_record_keys: set[str] = set() accepted_records = 0 effective_limit = max(0, int(limit)) if limit is not None else None logger.debug( "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s", run_id, lane, start_page, max_pages, ) try: params = self._build_search_params( search_url=search_url, make_id=make_id, model_id=model_id, price_min=price_min, price_max=price_max, year_min=year_min, year_max=year_max, mileage_min=mileage_min, mileage_max=mileage_max, sort_by=sort_by, sort_order=sort_order, ) def _on_page(page, meta: dict[str, int | None]) -> None: if progress_callback is None: return payload = { **meta, "page_url": page.url, "unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}), } progress_callback("page_collected", payload) pages_collected = 0 early_stopped = False concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1"))) if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True: page_iterator = self.client.fetch_search_pages_concurrent( start_page=start_page, max_pages=max_pages, workers=concurrent_pages, search_url=search_url, progress_callback=_on_page, **params, ) else: page_iterator = self.client.iter_search_pages( start_page=start_page, max_pages=max_pages, search_url=search_url, progress_callback=_on_page, **params, ) for page in page_iterator: pages_collected += 1 pages_payload.append(asdict(page)) listing_count += len(page.listings) unique_ids.update(str(listing.id) for listing in page.listings if listing.id) page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings] listing_by_record_key = { self._record_key(record): listing for listing, record in zip(page.listings, page_records, strict=False) if self._record_key(record) } page_records = self._dedupe_page_records(page_records, seen_record_keys) if runtime_filters is not None: page_records = [ record for record in page_records if runtime_filters.matches({ "brand": record.brand, "model": record.model, "year": record.year, "price": record.price, "mileage": record.mileage, "body_type": record.body_type, "color": record.color, "drive": record.drive, "gearbox": record.gearbox, "location": record.country, "is_damaged": record.is_damaged, }) ] if effective_limit is not None: page_records = page_records[:max(0, effective_limit - accepted_records)] for record in page_records: record.is_sold = False record.first_seen_at = run_seen_at record.last_seen_at = run_seen_at record.sold_at = None record.skip_image_sync = False existing_origin_ids: set[str] = set() if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED): existing_origin_ids = self.persistence.get_existing_origin_ids( [record.origin_id for record in page_records if record.origin_id] ) page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = ( self._apply_only_new_page_policy( page_records=page_records, only_new=only_new, sort_by=sort_by, sort_order=sort_order, skipped_existing=skipped_existing, existing_streak=existing_streak, new_records_kept=new_records_kept, existing_origin_ids=existing_origin_ids, ) ) if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids: for record in page_records: if record.origin_id and record.origin_id in existing_origin_ids: record.skip_image_sync = True if ( MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN and page_records ): remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget) candidate_records = [ record for record in page_records if record.origin_id and record.origin_id not in existing_origin_ids and self._record_needs_detail_enrich(record) ] candidate_records.sort(key=self._detail_enrich_priority, reverse=True) selected_keys = { self._record_key(record) for record in candidate_records[:page_budget] } if selected_keys: enriched_records: list[CarRecord] = [] for record in page_records: record_key = self._record_key(record) listing = listing_by_record_key.get(record_key) if record_key not in selected_keys or listing is None: enriched_records.append(record) continue try: if progress_callback is not None: progress_callback( "detail_enriching", { "run_id": run_id, "pages_collected": pages_collected, "page_number": page.page_number, "listing_id": str(listing.id), "detail_enriched": detail_enriched, "detail_enrich_failed": detail_enrich_failed, }, ) enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id)) enriched.is_sold = False enriched.first_seen_at = run_seen_at enriched.last_seen_at = run_seen_at enriched.sold_at = None enriched.skip_image_sync = False if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: enriched.images = record.images enriched_records.append(enriched) detail_enriched += 1 except Exception: logger.warning( "mobile.de selective detail enrich failed: listing_id=%s title=%s", getattr(listing, "id", None), getattr(listing, "title", None), exc_info=True, ) detail_enrich_failed += 1 enriched_records.append(record) page_records = enriched_records if progress_callback is not None: progress_callback( "records_mapped", { "record_count": len(page_records), "skipped_existing": skipped_existing, "detail_enriched": detail_enriched, "detail_enrich_failed": detail_enrich_failed, "only_new": bool(only_new), "run_id": run_id, "pages_collected": pages_collected, "page_number": page.page_number, }, ) upsert = self.persistence.upsert_cars_batch(page_records) if page_records else { "inserted": 0, "updated": 0, "images_upserted": 0, } page_inserted = int(upsert.get("inserted", 0)) page_updated = int(upsert.get("updated", 0)) page_images = int(upsert.get("images_upserted", 0)) ids_fetched += len(page_records) inserted_total += page_inserted updated_total += page_updated images_upserted += page_images cars_upserted = inserted_total + updated_total accepted_records += len(page_records) logger.debug( "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s", run_id, page.page_number, page_inserted, page_updated, page_images, ) if progress_callback is not None: progress_callback( "db_upsert_done", { "run_id": run_id, "pages_collected": pages_collected, "pages_in_batch": 1, "page_number": page.page_number, "inserted": page_inserted, "updated": page_updated, "images_upserted": page_images, }, ) if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit): early_stopped = True break data = { "source": "mobile.de", "strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE, "search_url": search_url, "pages": pages_payload, "listing_count": listing_count, "unique_listing_count": len(unique_ids), "unique_listing_ids": sorted(unique_ids), "early_stopped": early_stopped, } if progress_callback is not None: progress_callback( "search_collection_done", { "pages_collected": pages_collected, "listing_count": listing_count, "unique_listing_count": len(unique_ids), }, ) self.persistence.finish_sync_run( run_id, status="success", ids_fetched=ids_fetched, cars_upserted=cars_upserted, cars_failed=0, images_upserted=images_upserted, ) logger.debug( "mobile.de sync_search completed: run_id=%s listings=%s unique=%s", run_id, data.get("listing_count", 0), data.get("unique_listing_count", 0), ) return { "run_id": run_id, "upsert": { "inserted": inserted_total, "updated": updated_total, "images_upserted": images_upserted, }, "detail_enriched": detail_enriched, "detail_enrich_failed": detail_enrich_failed, "skipped_existing": skipped_existing, **data, } except Exception as exc: self.persistence.finish_sync_run( run_id, status="failed", ids_fetched=ids_fetched, cars_upserted=cars_upserted, cars_failed=0, images_upserted=images_upserted, error_summary=str(exc), ) logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True) raise def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]: self.persistence.create_tables() detail = self.client.fetch_detail(listing_id) record = self.mapper.detail_to_car_record(str(listing_id), detail) result = self.persistence.upsert_car(record) return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}