qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply runtime configuration

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:53 +03:00
parent fba7299747
commit 2d3260d8f9
4 changed files with 103 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
mobilede_scraper/mobile_de/scraper.py
View File

@@ -10,6 +10,7 @@ from typing import Any
import requests
from ..core.config import Settings, settings
from ..core.runtime_config import RuntimeFiltersConfig
from ..storage.db import PersistenceService
from ..storage.schemas import CarRecord
from .client import MobileDeClient
@@ -69,13 +70,12 @@ class MobileDeScraper:
sort_by: str | None,
sort_order: str | None,
) -> dict[str, str | None]:
params: dict[str, str | None] = {}
if not search_url:
params = {
params: dict[str, str | None] = {
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
}
if not search_url:
if make_id:
params["ms"] = self.client.build_make_model_param(make_id, model_id)
if sort_by:
@@ -325,7 +325,14 @@ class MobileDeScraper:
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
runtime_filters: RuntimeFiltersConfig | None = None,
limit: int | None = None,
) -> dict[str, Any]:
if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
raise ValueError(
"mobile.de does not support the runtime filter flags.run_and_drive: "
"the source payload has no reliable vehicle-condition field"
)
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
@@ -346,6 +353,8 @@ class MobileDeScraper:
new_records_kept = 0
head_cut_triggered = False
seen_record_keys: set[str] = set()
accepted_records = 0
effective_limit = max(0, int(limit)) if limit is not None else None
logger.debug(
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
run_id,
@@ -411,6 +420,25 @@ class MobileDeScraper:
if self._record_key(record)
}
page_records = self._dedupe_page_records(page_records, seen_record_keys)
if runtime_filters is not None:
page_records = [
record for record in page_records
if runtime_filters.matches({
"brand": record.brand,
"model": record.model,
"year": record.year,
"price": record.price,
"mileage": record.mileage,
"body_type": record.body_type,
"color": record.color,
"drive": record.drive,
"gearbox": record.gearbox,
"location": record.country,
"is_damaged": record.is_damaged,
})
]
if effective_limit is not None:
page_records = page_records[:max(0, effective_limit - accepted_records)]
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
@@ -528,6 +556,7 @@ class MobileDeScraper:
updated_total += page_updated
images_upserted += page_images
cars_upserted = inserted_total + updated_total
accepted_records += len(page_records)
logger.debug(
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
@@ -551,7 +580,7 @@ class MobileDeScraper:
},
)
if head_cut_triggered:
if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
early_stopped = True
break

15
mobilede_scraper/worker/search_sync.py
View File

@@ -423,7 +423,20 @@ def run_mobilede_sync_search_task(
scraper = MobileDeScraper(
client=MobileDeClient.for_worker(delay_seconds=delay_seconds),
persistence=_get_persistence(),
runtime_settings=settings,
)
runtime_filters = runtime_config.filters
if runtime_filters.price.min is not None:
price_min = str(runtime_filters.price.min)
if runtime_filters.price.max is not None:
price_max = str(runtime_filters.price.max)
if runtime_filters.mileage.min is not None:
mileage_min = str(runtime_filters.mileage.min)
if runtime_filters.mileage.max is not None:
mileage_max = str(runtime_filters.mileage.max)
if runtime_filters.include.years:
year_min = str(min(runtime_filters.include.years))
year_max = str(max(runtime_filters.include.years))
run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id)
result = scraper.sync_search(
start_page=actual_start_page,
@@ -443,6 +456,8 @@ def run_mobilede_sync_search_task(
mileage_max=mileage_max,
seen_at=run_seen_at,
progress_callback=_progress,
runtime_filters=runtime_filters,
limit=runtime_config.sync.limit,
)
inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0)
updated_count = int(result.get("upsert", {}).get("updated", 0) or 0)

29
tests/test_mobilede_runtime_filters.py Normal file
View File

@@ -0,0 +1,29 @@
from __future__ import annotations
import unittest
from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
class _Client:
def iter_search_pages(self, **kwargs):
del kwargs
return iter(())
class _Persistence:
def create_tables(self) -> None:
raise AssertionError("unsupported configuration must fail before persistence")
class TestMobileDeRuntimeFilters(unittest.TestCase):
def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
if __name__ == "__main__":
unittest.main()

27
tests/test_worker_runtime_tasks.py
View File

@@ -523,7 +523,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
@@ -543,12 +543,22 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
if previous is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
@@ -556,20 +566,27 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
]
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
if previous_fast_start is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
self.assertGreater(len(segments), 300)
self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments))
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))