qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply runtime configuration

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:53 +03:00
parent fba7299747
commit 2d3260d8f9
4 changed files with 103 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
tests/test_mobilede_runtime_filters.py Normal file
View File

@@ -0,0 +1,29 @@
from __future__ import annotations
import unittest
from mobilede_scraper.core.runtime_config import RuntimeFiltersConfig
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
class _Client:
def iter_search_pages(self, **kwargs):
del kwargs
return iter(())
class _Persistence:
def create_tables(self) -> None:
raise AssertionError("unsupported configuration must fail before persistence")
class TestMobileDeRuntimeFilters(unittest.TestCase):
def test_run_and_drive_filter_is_rejected_without_reliable_source_field(self) -> None:
filters = RuntimeFiltersConfig.from_dict({"flags": {"run_and_drive": True}})
with self.assertRaisesRegex(ValueError, r"mobile\.de does not support.*run_and_drive"):
MobileDeScraper(client=_Client(), persistence=_Persistence()).sync_search(runtime_filters=filters)
if __name__ == "__main__":
unittest.main()

29
tests/test_worker_runtime_tasks.py
View File

@@ -523,7 +523,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
@@ -543,12 +543,22 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
segments = tasks._build_mobilede_runtime_segments(settings)
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
if previous is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
@@ -556,20 +566,27 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222"
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
]
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
if previous_fast_start is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
self.assertGreater(len(segments), 300)
self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments))
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))