Apply runtime configuration

mobilede_scraper/mobile_de/scraper.py

@@ -10,6 +10,7 @@ from typing import Any
 import requests
 
 from ..core.config import Settings, settings
+from ..core.runtime_config import RuntimeFiltersConfig
 from ..storage.db import PersistenceService
 from ..storage.schemas import CarRecord
 from .client import MobileDeClient
@@ -69,13 +70,12 @@ class MobileDeScraper:
         sort_by: str | None,
         sort_order: str | None,
     ) -> dict[str, str | None]:
-        params: dict[str, str | None] = {}
+        params: dict[str, str | None] = {
+            "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
+            "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
+            "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
+        }
         if not search_url:
-            params = {
-                "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
-                "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
-                "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
-            }
             if make_id:
                 params["ms"] = self.client.build_make_model_param(make_id, model_id)
         if sort_by:
@@ -325,7 +325,14 @@ class MobileDeScraper:
         sort_order: str | None = None,
         seen_at: datetime | None = None,
         progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
+        runtime_filters: RuntimeFiltersConfig | None = None,
+        limit: int | None = None,
     ) -> dict[str, Any]:
+        if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
+            raise ValueError(
+                "mobile.de does not support the runtime filter flags.run_and_drive: "
+                "the source payload has no reliable vehicle-condition field"
+            )
         self.persistence.create_tables()
         run_id = self.persistence.start_sync_run(lane)
         run_seen_at = seen_at or datetime.now(timezone.utc)
@@ -346,6 +353,8 @@ class MobileDeScraper:
         new_records_kept = 0
         head_cut_triggered = False
         seen_record_keys: set[str] = set()
+        accepted_records = 0
+        effective_limit = max(0, int(limit)) if limit is not None else None
         logger.debug(
             "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
             run_id,
@@ -411,6 +420,25 @@ class MobileDeScraper:
                     if self._record_key(record)
                 }
                 page_records = self._dedupe_page_records(page_records, seen_record_keys)
+                if runtime_filters is not None:
+                    page_records = [
+                        record for record in page_records
+                        if runtime_filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "location": record.country,
+                            "is_damaged": record.is_damaged,
+                        })
+                    ]
+                if effective_limit is not None:
+                    page_records = page_records[:max(0, effective_limit - accepted_records)]
                 for record in page_records:
                     record.is_sold = False
                     record.first_seen_at = run_seen_at
@@ -528,6 +556,7 @@ class MobileDeScraper:
                 updated_total += page_updated
                 images_upserted += page_images
                 cars_upserted = inserted_total + updated_total
+                accepted_records += len(page_records)
 
                 logger.debug(
                     "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
@@ -551,7 +580,7 @@ class MobileDeScraper:
                         },
                     )
 
-                if head_cut_triggered:
+                if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
                     early_stopped = True
                     break
 

mobilede_scraper/worker/search_sync.py

@@ -423,7 +423,20 @@ def run_mobilede_sync_search_task(
         scraper = MobileDeScraper(
             client=MobileDeClient.for_worker(delay_seconds=delay_seconds),
             persistence=_get_persistence(),
+            runtime_settings=settings,
         )
+        runtime_filters = runtime_config.filters
+        if runtime_filters.price.min is not None:
+            price_min = str(runtime_filters.price.min)
+        if runtime_filters.price.max is not None:
+            price_max = str(runtime_filters.price.max)
+        if runtime_filters.mileage.min is not None:
+            mileage_min = str(runtime_filters.mileage.min)
+        if runtime_filters.mileage.max is not None:
+            mileage_max = str(runtime_filters.mileage.max)
+        if runtime_filters.include.years:
+            year_min = str(min(runtime_filters.include.years))
+            year_max = str(max(runtime_filters.include.years))
         run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id)
         result = scraper.sync_search(
             start_page=actual_start_page,
@@ -443,6 +456,8 @@ def run_mobilede_sync_search_task(
             mileage_max=mileage_max,
             seen_at=run_seen_at,
             progress_callback=_progress,
+            runtime_filters=runtime_filters,
+            limit=runtime_config.sync.limit,
         )
         inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0)
         updated_count = int(result.get("upsert", {}).get("updated", 0) or 0)

tests/test_mobilede_runtime_filters.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import unittest
+
+from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
+from mobilede_scraper.mobile_de.scraper import MobileDeScraper
+
+
+class _Client:
+    def iter_search_pages(self, **kwargs):
+        del kwargs
+        return iter(())
+
+
+class _Persistence:
+    def create_tables(self) -> None:
+        raise AssertionError("unsupported configuration must fail before persistence")
+
+
+class TestMobileDeRuntimeFilters(unittest.TestCase):
+    def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
+        filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
+
+        with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
+            MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_worker_runtime_tasks.py

@@ -523,7 +523,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
 
         settings = MagicMock()
         settings.listing.filtered_search_urls = [
-            "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
         ]
 
         previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
@@ -543,12 +543,22 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
 
     def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
+        import os
+
         settings = MagicMock()
         settings.listing.filtered_search_urls = [
-            "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
         ]
 
-        segments = tasks._build_mobilede_runtime_segments(settings)
+        previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
+        try:
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
+            segments = tasks._build_mobilede_runtime_segments(settings)
+        finally:
+            if previous is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
 
         self.assertGreater(len(segments), 2)
         self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
@@ -556,20 +566,27 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
         self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
 
     def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
+        import os
+
         settings = MagicMock()
         settings.listing.filtered_search_urls = [
-            "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222"
+            "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
         ]
 
         original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
+        previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
         try:
             tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
+            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
             segments = tasks._build_mobilede_runtime_segments(settings)
         finally:
             tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
+            if previous_fast_start is None:
+                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
+            else:
+                os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
 
-        self.assertGreater(len(segments), 300)
+        self.assertGreater(len(segments), 2)
-        self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments))
         self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
         self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))