refactor mobile.de parser, fix country mapping, update README
This commit is contained in:
qananasikq
471
tests/test_worker_runtime_tasks.py
Normal file
471
tests/test_worker_runtime_tasks.py
Normal file
@@ -0,0 +1,471 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from mobilede_scraper.worker import tasks
|
||||
|
||||
|
||||
class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
|
||||
class _FakeRedis:
|
||||
def __init__(self) -> None:
|
||||
self.store: dict[str, str] = {}
|
||||
self.sets: dict[str, set[str]] = {}
|
||||
|
||||
def get(self, key: str):
|
||||
return self.store.get(key)
|
||||
|
||||
def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002
|
||||
if nx and key in self.store:
|
||||
return False
|
||||
self.store[key] = str(value)
|
||||
return True
|
||||
|
||||
def incr(self, key: str):
|
||||
current = int(self.store.get(key, "0"))
|
||||
current += 1
|
||||
self.store[key] = str(current)
|
||||
return current
|
||||
|
||||
def sadd(self, key: str, member: str):
|
||||
bucket = self.sets.setdefault(key, set())
|
||||
before = len(bucket)
|
||||
bucket.add(member)
|
||||
return 1 if len(bucket) > before else 0
|
||||
|
||||
def expire(self, key: str, ttl: int): # noqa: ARG002
|
||||
return True
|
||||
|
||||
def test_lock_acquire_refresh_release(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
redis_client.set.return_value = True
|
||||
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
||||
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
||||
|
||||
redis_client.eval.return_value = 1
|
||||
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
||||
|
||||
redis_client.eval.reset_mock()
|
||||
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
||||
args = redis_client.eval.call_args[0]
|
||||
self.assertEqual(args[2], "lock:key")
|
||||
self.assertEqual(args[3], "owner-token")
|
||||
|
||||
def test_mobilede_segment_key_is_stable(self) -> None:
|
||||
segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000}
|
||||
|
||||
key1 = tasks._mobilede_task_segment_key(
|
||||
segment=segment,
|
||||
search_url=None,
|
||||
make_id=None,
|
||||
model_id=None,
|
||||
price_min=None,
|
||||
price_max=None,
|
||||
year_min=None,
|
||||
year_max=None,
|
||||
mileage_min=None,
|
||||
mileage_max=None,
|
||||
)
|
||||
key2 = tasks._mobilede_task_segment_key(
|
||||
segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000},
|
||||
search_url=None,
|
||||
make_id=None,
|
||||
model_id=None,
|
||||
price_min=None,
|
||||
price_max=None,
|
||||
year_min=None,
|
||||
year_max=None,
|
||||
mileage_min=None,
|
||||
mileage_max=None,
|
||||
)
|
||||
|
||||
self.assertEqual(key1, key2)
|
||||
self.assertEqual(len(key1), 16)
|
||||
|
||||
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
settings = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]',
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2",
|
||||
}.get(key)
|
||||
redis_client.incr.return_value = 2
|
||||
redis_client.ttl.return_value = -2
|
||||
|
||||
segment = tasks._reserve_mobilede_runtime_segment(
|
||||
redis_client,
|
||||
settings,
|
||||
only_new=False,
|
||||
)
|
||||
|
||||
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
|
||||
|
||||
def test_preplan_splits_dense_segment_before_queueing(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_children = tasks._mobilede_build_overflow_child_segments
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
|
||||
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
|
||||
|
||||
parent = {
|
||||
"label": "Cars",
|
||||
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
|
||||
child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
|
||||
tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
|
||||
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments([parent])
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks._mobilede_build_overflow_child_segments = original_children
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
|
||||
|
||||
self.assertEqual(len(planned), 2)
|
||||
self.assertTrue(all(item.get("total_results") == 100 for item in planned))
|
||||
self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
|
||||
|
||||
def test_preplan_has_hard_segment_limit(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_children = tasks._mobilede_build_overflow_child_segments
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
|
||||
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
|
||||
|
||||
parent = {
|
||||
"label": "Cars",
|
||||
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
|
||||
tasks._mobilede_probe_segment_total = lambda segment: 5000
|
||||
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
|
||||
{**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
|
||||
{**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
|
||||
]
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments([parent])
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks._mobilede_build_overflow_child_segments = original_children
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
|
||||
|
||||
self.assertGreaterEqual(len(planned), 1)
|
||||
self.assertLessEqual(len(planned), 3)
|
||||
|
||||
def test_preplan_probe_budget_is_global(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
|
||||
calls = {"count": 0}
|
||||
segments = [
|
||||
{"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"}
|
||||
for index in range(5)
|
||||
]
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2
|
||||
|
||||
def _probe(_segment):
|
||||
calls["count"] += 1
|
||||
return 100
|
||||
|
||||
tasks._mobilede_probe_segment_total = _probe
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments(segments)
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
|
||||
self.assertEqual(calls["count"], 2)
|
||||
self.assertEqual(len(planned), 5)
|
||||
|
||||
def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
|
||||
original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
|
||||
try:
|
||||
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
|
||||
children = tasks._mobilede_build_overflow_child_segments(
|
||||
segment={
|
||||
"label": "Cars",
|
||||
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
|
||||
"price_min": "5001",
|
||||
"price_max": "10000",
|
||||
"year_min": "2005",
|
||||
"year_max": "2009",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
},
|
||||
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
|
||||
|
||||
self.assertEqual(len(children), 2)
|
||||
self.assertEqual(children[0].get("mileage_max"), "150000")
|
||||
self.assertEqual(children[1].get("mileage_min"), "150001")
|
||||
self.assertIsNone(children[1].get("mileage_max"))
|
||||
|
||||
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
||||
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
||||
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
||||
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
||||
}.get(key)
|
||||
redis_client.scard.return_value = 1
|
||||
|
||||
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
||||
|
||||
self.assertFalse(finalized)
|
||||
redis_client.set.assert_not_called()
|
||||
|
||||
def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
||||
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
||||
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
||||
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
||||
}.get(key)
|
||||
redis_client.scard.return_value = 0
|
||||
|
||||
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
||||
|
||||
self.assertTrue(finalized)
|
||||
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
|
||||
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
|
||||
|
||||
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
|
||||
|
||||
should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment(
|
||||
redis_client,
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
only_new=False,
|
||||
bootstrap_run_active=True,
|
||||
)
|
||||
|
||||
self.assertTrue(should_skip)
|
||||
|
||||
def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
|
||||
exhausted = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=51,
|
||||
max_pages=50,
|
||||
use_cursor=False,
|
||||
)
|
||||
not_exhausted = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=1,
|
||||
max_pages=50,
|
||||
use_cursor=False,
|
||||
)
|
||||
cursor_window = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=51,
|
||||
max_pages=50,
|
||||
use_cursor=True,
|
||||
)
|
||||
|
||||
self.assertTrue(exhausted)
|
||||
self.assertFalse(not_exhausted)
|
||||
self.assertFalse(cursor_window)
|
||||
|
||||
def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None:
|
||||
segment = {"make_id": "3500", "start_page": 1}
|
||||
|
||||
requested_start_page = 51
|
||||
if segment.get("start_page") is not None and requested_start_page <= 1:
|
||||
effective_start_page = int(segment.get("start_page") or requested_start_page)
|
||||
else:
|
||||
effective_start_page = requested_start_page
|
||||
|
||||
self.assertEqual(effective_start_page, 51)
|
||||
|
||||
def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None:
|
||||
redis_client = self._FakeRedis()
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1"
|
||||
|
||||
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
||||
redis_client,
|
||||
cycle_id="cycle-a",
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
total_segments_hint=2,
|
||||
)
|
||||
|
||||
self.assertEqual(done, 2)
|
||||
self.assertEqual(total, 2)
|
||||
self.assertTrue(should_finalize)
|
||||
|
||||
def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None:
|
||||
redis_client = self._FakeRedis()
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0"
|
||||
|
||||
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
||||
redis_client,
|
||||
cycle_id="cycle-old",
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
total_segments_hint=2,
|
||||
)
|
||||
|
||||
self.assertEqual(done, 0)
|
||||
self.assertEqual(total, 2)
|
||||
self.assertFalse(should_finalize)
|
||||
|
||||
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
|
||||
self.assertFalse(
|
||||
tasks._mobilede_should_pre_split_mileage(
|
||||
price_min=30001,
|
||||
price_max=50000,
|
||||
year_min=2010,
|
||||
year_max=2014,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
|
||||
def test_pre_split_mileage_respects_explicit_override(self) -> None:
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
|
||||
self.assertTrue(
|
||||
tasks._mobilede_should_pre_split_mileage(
|
||||
price_min=30001,
|
||||
price_max=50000,
|
||||
year_min=2010,
|
||||
year_max=2017,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
|
||||
def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
|
||||
search_url = (
|
||||
"https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
|
||||
"?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
|
||||
)
|
||||
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
|
||||
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
|
||||
segments = tasks._expand_mobilede_search_url_segment(
|
||||
{
|
||||
"label": "Cars",
|
||||
"search_url": search_url,
|
||||
"start_page": 1,
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
|
||||
|
||||
make_counts: dict[str, int] = {}
|
||||
for segment in segments:
|
||||
make_id = str(segment.get("make_id") or "")
|
||||
make_counts[make_id] = make_counts.get(make_id, 0) + 1
|
||||
self.assertIsNone(segment.get("mileage_min"))
|
||||
self.assertIsNone(segment.get("mileage_max"))
|
||||
|
||||
self.assertEqual(make_counts, {"3500": 59, "11000": 59})
|
||||
self.assertEqual(len(segments), 118)
|
||||
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
|
||||
|
||||
|
||||
def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
|
||||
settings = MagicMock()
|
||||
settings.listing.filtered_search_urls = [
|
||||
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
|
||||
]
|
||||
|
||||
segments = tasks._build_mobilede_runtime_segments(settings)
|
||||
|
||||
self.assertEqual(len(segments), 118)
|
||||
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
|
||||
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
|
||||
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
|
||||
import os
|
||||
|
||||
settings = MagicMock()
|
||||
settings.listing.filtered_search_urls = [
|
||||
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
|
||||
]
|
||||
|
||||
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
|
||||
try:
|
||||
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true"
|
||||
segments = tasks._build_mobilede_runtime_segments(settings)
|
||||
finally:
|
||||
if previous is None:
|
||||
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
|
||||
else:
|
||||
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
|
||||
|
||||
self.assertEqual(len(segments), 2)
|
||||
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
|
||||
self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user