from __future__ import annotations import unittest from unittest.mock import MagicMock from mobilede_scraper.worker import tasks class TestWorkerRuntimeTaskHelpers(unittest.TestCase): class _FakeRedis: def __init__(self) -> None: self.store: dict[str, str] = {} self.sets: dict[str, set[str]] = {} def get(self, key: str): return self.store.get(key) def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002 if nx and key in self.store: return False self.store[key] = str(value) return True def incr(self, key: str): current = int(self.store.get(key, "0")) current += 1 self.store[key] = str(current) return current def sadd(self, key: str, member: str): bucket = self.sets.setdefault(key, set()) before = len(bucket) bucket.add(member) return 1 if len(bucket) > before else 0 def expire(self, key: str, ttl: int): # noqa: ARG002 return True def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() redis_client.set.return_value = True self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) redis_client.eval.return_value = 1 self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) redis_client.eval.reset_mock() tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") args = redis_client.eval.call_args[0] self.assertEqual(args[2], "lock:key") self.assertEqual(args[3], "owner-token") def test_mobilede_segment_key_is_stable(self) -> None: segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000} key1 = tasks._mobilede_task_segment_key( segment=segment, search_url=None, make_id=None, model_id=None, price_min=None, price_max=None, year_min=None, year_max=None, mileage_min=None, mileage_max=None, ) key2 = tasks._mobilede_task_segment_key( segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000}, search_url=None, make_id=None, model_id=None, price_min=None, price_max=None, year_min=None, year_max=None, mileage_min=None, mileage_max=None, ) self.assertEqual(key1, key2) self.assertEqual(len(key1), 16) def test_runtime_segment_reservation_uses_pending_cache(self) -> None: redis_client = MagicMock() settings = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]', tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2", }.get(key) redis_client.incr.return_value = 2 redis_client.ttl.return_value = -2 segment = tasks._reserve_mobilede_runtime_segment( redis_client, settings, only_new=False, ) self.assertEqual(segment, (1, {"make_id": "AUDI"})) def test_preplan_splits_dense_segment_before_queueing(self) -> None: original_probe = tasks._mobilede_probe_segment_total original_children = tasks._mobilede_build_overflow_child_segments original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH parent = { "label": "Cars", "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, } child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1} child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1} try: tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450 tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700 tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100 tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b] planned = tasks._mobilede_preplan_runtime_segments([parent]) finally: tasks._mobilede_probe_segment_total = original_probe tasks._mobilede_build_overflow_child_segments = original_children tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth self.assertEqual(len(planned), 2) self.assertTrue(all(item.get("total_results") == 100 for item in planned)) self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned)) def test_preplan_has_hard_segment_limit(self) -> None: original_probe = tasks._mobilede_probe_segment_total original_children = tasks._mobilede_build_overflow_child_segments original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH parent = { "label": "Cars", "search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true", "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, } try: tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3 tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10 tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0 tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2 tasks._mobilede_probe_segment_total = lambda segment: 5000 tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [ {**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, {**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1}, ] planned = tasks._mobilede_preplan_runtime_segments([parent]) finally: tasks._mobilede_probe_segment_total = original_probe tasks._mobilede_build_overflow_child_segments = original_children tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth self.assertGreaterEqual(len(planned), 1) self.assertLessEqual(len(planned), 3) def test_preplan_probe_budget_is_global(self) -> None: original_probe = tasks._mobilede_probe_segment_total original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES calls = {"count": 0} segments = [ {"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"} for index in range(5) ] try: tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50 tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2 def _probe(_segment): calls["count"] += 1 return 100 tasks._mobilede_probe_segment_total = _probe planned = tasks._mobilede_preplan_runtime_segments(segments) finally: tasks._mobilede_probe_segment_total = original_probe tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes self.assertEqual(calls["count"], 2) self.assertEqual(len(planned), 5) def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None: original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS try: tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2 children = tasks._mobilede_build_overflow_child_segments( segment={ "label": "Cars", "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009", "price_min": "5001", "price_max": "10000", "year_min": "2005", "year_max": "2009", "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, }, max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER, ) finally: tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children self.assertEqual(len(children), 2) self.assertEqual(children[0].get("mileage_max"), "150000") self.assertEqual(children[1].get("mileage_min"), "150001") self.assertIsNone(children[1].get("mileage_max")) def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", }.get(key) redis_client.scard.return_value = 1 finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) self.assertFalse(finalized) redis_client.set.assert_not_called() def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", }.get(key) redis_client.scard.return_value = 0 finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) self.assertTrue(finalized) redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1") redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60) def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment( redis_client, segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, only_new=False, bootstrap_run_active=True, ) self.assertTrue(should_skip) def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None: exhausted = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=51, max_pages=50, use_cursor=False, ) not_exhausted = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=1, max_pages=50, use_cursor=False, ) cursor_window = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=51, max_pages=50, use_cursor=True, ) self.assertTrue(exhausted) self.assertFalse(not_exhausted) self.assertFalse(cursor_window) def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None: segment = {"make_id": "3500", "start_page": 1} requested_start_page = 51 if segment.get("start_page") is not None and requested_start_page <= 1: effective_start_page = int(segment.get("start_page") or requested_start_page) else: effective_start_page = requested_start_page self.assertEqual(effective_start_page, 51) def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1" done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( redis_client, cycle_id="cycle-a", segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, total_segments_hint=2, ) self.assertEqual(done, 2) self.assertEqual(total, 2) self.assertTrue(should_finalize) def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0" done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( redis_client, cycle_id="cycle-old", segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, total_segments_hint=2, ) self.assertEqual(done, 0) self.assertEqual(total, 2) self.assertFalse(should_finalize) def test_pre_split_mileage_requires_explicit_flag(self) -> None: original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False self.assertFalse( tasks._mobilede_should_pre_split_mileage( price_min=30001, price_max=50000, year_min=2010, year_max=2014, ) ) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split def test_pre_split_mileage_respects_explicit_override(self) -> None: original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True self.assertTrue( tasks._mobilede_should_pre_split_mileage( price_min=30001, price_max=50000, year_min=2010, year_max=2017, ) ) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None: search_url = ( "https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html" "?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp" ) original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False segments = tasks._expand_mobilede_search_url_segment( { "label": "Cars", "search_url": search_url, "start_page": 1, "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, } ) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic make_counts: dict[str, int] = {} for segment in segments: make_id = str(segment.get("make_id") or "") make_counts[make_id] = make_counts.get(make_id, 0) + 1 self.assertIsNone(segment.get("mileage_min")) self.assertIsNone(segment.get("mileage_max")) self.assertEqual(make_counts, {"3500": 59, "11000": 59}) self.assertEqual(len(segments), 118) self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments)) def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None: settings = MagicMock() settings.listing.filtered_search_urls = [ "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" ] segments = tasks._build_mobilede_runtime_segments(settings) self.assertEqual(len(segments), 118) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None: import os settings = MagicMock() settings.listing.filtered_search_urls = [ "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000" ] previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") try: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true" segments = tasks._build_mobilede_runtime_segments(settings) finally: if previous is None: os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) else: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous self.assertEqual(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) if __name__ == "__main__": unittest.main()