From 2d3260d8f97a209d339e59418b05bc13eb8ab878 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 5 Aug 2026 18:13:53 +0300 Subject: [PATCH] Apply runtime configuration --- mobilede_scraper/mobile_de/scraper.py | 43 +++++++++++++++++++++----- mobilede_scraper/worker/search_sync.py | 15 +++++++++ tests/test_mobilede_runtime_filters.py | 29 +++++++++++++++++ tests/test_worker_runtime_tasks.py | 29 +++++++++++++---- 4 files changed, 103 insertions(+), 13 deletions(-) create mode 100644 tests/test_mobilede_runtime_filters.py diff --git a/mobilede_scraper/mobile_de/scraper.py b/mobilede_scraper/mobile_de/scraper.py index 2b257fa..65dc518 100644 --- a/mobilede_scraper/mobile_de/scraper.py +++ b/mobilede_scraper/mobile_de/scraper.py @@ -10,6 +10,7 @@ from typing import Any import requests from ..core.config import Settings, settings +from ..core.runtime_config import RuntimeFiltersConfig from ..storage.db import PersistenceService from ..storage.schemas import CarRecord from .client import MobileDeClient @@ -69,13 +70,12 @@ class MobileDeScraper: sort_by: str | None, sort_order: str | None, ) -> dict[str, str | None]: - params: dict[str, str | None] = {} + params: dict[str, str | None] = { + "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, + "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, + "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, + } if not search_url: - params = { - "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, - "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, - "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, - } if make_id: params["ms"] = self.client.build_make_model_param(make_id, model_id) if sort_by: @@ -325,7 +325,14 @@ class MobileDeScraper: sort_order: str | None = None, seen_at: datetime | None = None, progress_callback: Callable[[str, dict[str, Any]], None] | None = None, + runtime_filters: RuntimeFiltersConfig | None = None, + limit: int | None = None, ) -> dict[str, Any]: + if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None: + raise ValueError( + "mobile.de does not support the runtime filter flags.run_and_drive: " + "the source payload has no reliable vehicle-condition field" + ) self.persistence.create_tables() run_id = self.persistence.start_sync_run(lane) run_seen_at = seen_at or datetime.now(timezone.utc) @@ -346,6 +353,8 @@ class MobileDeScraper: new_records_kept = 0 head_cut_triggered = False seen_record_keys: set[str] = set() + accepted_records = 0 + effective_limit = max(0, int(limit)) if limit is not None else None logger.debug( "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s", run_id, @@ -411,6 +420,25 @@ class MobileDeScraper: if self._record_key(record) } page_records = self._dedupe_page_records(page_records, seen_record_keys) + if runtime_filters is not None: + page_records = [ + record for record in page_records + if runtime_filters.matches({ + "brand": record.brand, + "model": record.model, + "year": record.year, + "price": record.price, + "mileage": record.mileage, + "body_type": record.body_type, + "color": record.color, + "drive": record.drive, + "gearbox": record.gearbox, + "location": record.country, + "is_damaged": record.is_damaged, + }) + ] + if effective_limit is not None: + page_records = page_records[:max(0, effective_limit - accepted_records)] for record in page_records: record.is_sold = False record.first_seen_at = run_seen_at @@ -528,6 +556,7 @@ class MobileDeScraper: updated_total += page_updated images_upserted += page_images cars_upserted = inserted_total + updated_total + accepted_records += len(page_records) logger.debug( "mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s", @@ -551,7 +580,7 @@ class MobileDeScraper: }, ) - if head_cut_triggered: + if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit): early_stopped = True break diff --git a/mobilede_scraper/worker/search_sync.py b/mobilede_scraper/worker/search_sync.py index 44d683b..1307601 100644 --- a/mobilede_scraper/worker/search_sync.py +++ b/mobilede_scraper/worker/search_sync.py @@ -423,7 +423,20 @@ def run_mobilede_sync_search_task( scraper = MobileDeScraper( client=MobileDeClient.for_worker(delay_seconds=delay_seconds), persistence=_get_persistence(), + runtime_settings=settings, ) + runtime_filters = runtime_config.filters + if runtime_filters.price.min is not None: + price_min = str(runtime_filters.price.min) + if runtime_filters.price.max is not None: + price_max = str(runtime_filters.price.max) + if runtime_filters.mileage.min is not None: + mileage_min = str(runtime_filters.mileage.min) + if runtime_filters.mileage.max is not None: + mileage_max = str(runtime_filters.mileage.max) + if runtime_filters.include.years: + year_min = str(min(runtime_filters.include.years)) + year_max = str(max(runtime_filters.include.years)) run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id) result = scraper.sync_search( start_page=actual_start_page, @@ -443,6 +456,8 @@ def run_mobilede_sync_search_task( mileage_max=mileage_max, seen_at=run_seen_at, progress_callback=_progress, + runtime_filters=runtime_filters, + limit=runtime_config.sync.limit, ) inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) updated_count = int(result.get("upsert", {}).get("updated", 0) or 0) diff --git a/tests/test_mobilede_runtime_filters.py b/tests/test_mobilede_runtime_filters.py new file mode 100644 index 0000000..4ca92b3 --- /dev/null +++ b/tests/test_mobilede_runtime_filters.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import unittest + +from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig +from mobilede_scraper.mobile_de.scraper import MobileDeScraper + + +class _Client: + def iter_search_pages(self, **kwargs): + del kwargs + return iter(()) + + +class _Persistence: + def create_tables(self) -> None: + raise AssertionError("unsupported configuration must fail before persistence") + + +class TestMobileDeRuntimeFilters(unittest.TestCase): + def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None: + filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}}) + + with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"): + MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_runtime_tasks.py b/tests/test_worker_runtime_tasks.py index 42765ff..92a1986 100644 --- a/tests/test_worker_runtime_tasks.py +++ b/tests/test_worker_runtime_tasks.py @@ -523,7 +523,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): settings = MagicMock() settings.listing.filtered_search_urls = [ - "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" + "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en" ] previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") @@ -543,12 +543,22 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None: + import os + settings = MagicMock() settings.listing.filtered_search_urls = [ - "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" + "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en" ] - segments = tasks._build_mobilede_runtime_segments(settings) + previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") + try: + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false" + segments = tasks._build_mobilede_runtime_segments(settings) + finally: + if previous is None: + os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) + else: + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous self.assertGreater(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) @@ -556,20 +566,27 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase): self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments)) def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None: + import os + settings = MagicMock() settings.listing.filtered_search_urls = [ - "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222" + "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en" ] original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE + previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false" segments = tasks._build_mobilede_runtime_segments(settings) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split + if previous_fast_start is None: + os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) + else: + os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start - self.assertGreater(len(segments), 300) - self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments)) + self.assertGreater(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"}) self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))