from __future__ import annotations from datetime import datetime, timezone from types import SimpleNamespace import unittest from unittest.mock import MagicMock, patch from mobilede_scraper.worker import tasks class TestWorkerRuntimeTaskHelpers(unittest.TestCase): class _FakeRedis: def __init__(self) -> None: self.store: dict[str, str] = {} self.sets: dict[str, set[str]] = {} self.lists: dict[str, list[str]] = {} def get(self, key: str): return self.store.get(key) def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002 if nx and key in self.store: return False self.store[key] = str(value) return True def incr(self, key: str): current = int(self.store.get(key, "0")) current += 1 self.store[key] = str(current) return current def sadd(self, key: str, member: str): bucket = self.sets.setdefault(key, set()) before = len(bucket) bucket.add(member) return 1 if len(bucket) > before else 0 def scard(self, key: str): return len(self.sets.get(key, set())) def llen(self, key: str): return len(self.lists.get(key, [])) def delete(self, *keys: str): removed = 0 for key in keys: if key in self.store: del self.store[key] removed += 1 if key in self.sets: del self.sets[key] removed += 1 if key in self.lists: del self.lists[key] removed += 1 return removed def expire(self, key: str, ttl: int): # noqa: ARG002 return True def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() redis_client.set.return_value = True self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) redis_client.eval.return_value = 1 self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) redis_client.eval.reset_mock() tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") args = redis_client.eval.call_args[0] self.assertEqual(args[2], "lock:key") self.assertEqual(args[3], "owner-token") def test_mobilede_segment_key_is_stable(self) -> None: segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000} key1 = tasks._mobilede_task_segment_key( segment=segment, search_url=None, make_id=None, model_id=None, price_min=None, price_max=None, year_min=None, year_max=None, mileage_min=None, mileage_max=None, ) key2 = tasks._mobilede_task_segment_key( segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000}, search_url=None, make_id=None, model_id=None, price_min=None, price_max=None, year_min=None, year_max=None, mileage_min=None, mileage_max=None, ) self.assertEqual(key1, key2) self.assertEqual(len(key1), 16) def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None: parent = { "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000", "make_id": "3500", "price_min": "1", "price_max": "5000", "year_max": "2004", "mileage_max": "100000", } parent_key = tasks._mobilede_segment_fingerprint(parent) child_a = { **parent, "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000", "mileage_max": "50000", "overflow_parent": parent_key, } child_b = { **parent, "search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000", "mileage_min": "50001", "mileage_max": "100000", "overflow_parent": parent_key, } pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b]) self.assertEqual(pruned, [child_a, child_b]) def test_runtime_segment_reservation_uses_pending_cache(self) -> None: redis_client = MagicMock() settings = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]', tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2", }.get(key) redis_client.incr.return_value = 2 redis_client.ttl.return_value = -2 segment = tasks._reserve_mobilede_runtime_segment( redis_client, settings, only_new=False, ) self.assertEqual(segment, (1, {"make_id": "AUDI"})) def test_preplan_probe_budget_is_global(self) -> None: original_probe = tasks._mobilede_probe_segment_total original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES calls = {"count": 0} segments = [ {"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"} for index in range(5) ] try: tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50 tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2 def _probe(_segment): calls["count"] += 1 return 100 tasks._mobilede_probe_segment_total = _probe planned = tasks._mobilede_preplan_runtime_segments(segments) finally: tasks._mobilede_probe_segment_total = original_probe tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes self.assertEqual(calls["count"], 2) self.assertEqual(len(planned), 5) def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", }.get(key) redis_client.scard.return_value = 1 finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) self.assertFalse(finalized) redis_client.set.assert_not_called() def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: { tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10", tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None, tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000", tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995", tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990", tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5", tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000", }.get(key) redis_client.scard.return_value = 0 finalized = tasks._mobilede_try_finalize_bootstrap(redis_client) self.assertTrue(finalized) redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1") redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60) def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10" redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10" effective_only_new = tasks._mobilede_force_full_scan_only_new( True, redis_client=redis_client, continuous=True, ) self.assertFalse(effective_only_new) def test_post_bootstrap_refresh_disabled_for_continuous_hourly_cycle(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10" redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10" post_bootstrap_refresh = tasks._mobilede_post_bootstrap_full_refresh( redis_client, False, continuous=True, ) self.assertFalse(post_bootstrap_refresh) def test_stalled_bootstrap_recovery_ignores_stale_global_progress_key(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13" redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261" redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "1" redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"} with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000): recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client) self.assertTrue(recovered) self.assertNotIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets) def test_stalled_bootstrap_recovery_keeps_recent_progress(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13" redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261" redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "9950" redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"} with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000): recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client) self.assertFalse(recovered) self.assertIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets) def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None: redis_client = self._FakeRedis() started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc) redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat() redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141" persistence = MagicMock() persistence.mark_sold_not_seen_since.return_value = 42 with ( patch.object(tasks, "_get_persistence", return_value=persistence), patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False), ): finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion( redis_client, cycle_id="cycle-lag", total_segments_hint=377, ) self.assertTrue(finalized) self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377") persistence.mark_sold_not_seen_since.assert_called_once_with( started_at, prefix=("mobile.de:", "mobilede:"), safety_ratio=0.8, ) def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None: redis_client = MagicMock() redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment( redis_client, segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, only_new=False, bootstrap_run_active=True, ) self.assertTrue(should_skip) def test_queue_mobilede_bootstrap_recovery_can_force_requeue_existing_pending(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1" with patch.object(tasks.mobilede_sync_runtime_segments_task, "apply_async") as apply_async: queued = tasks._queue_mobilede_bootstrap_recovery( redis_client, lane="mobile_de_cars", delay_seconds=0.7, use_cursor=True, continuous=True, segment_label="runtime_segments", reason="waiting_for_active_bootstrap_tasks", countdown=5, force=True, ) self.assertTrue(queued) apply_async.assert_called_once() self.assertEqual(redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY], "1") def test_runtime_segments_task_requeues_pending_bootstrap_recovery_while_tasks_are_still_active(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1" runtime_config = SimpleNamespace(sync=SimpleNamespace(only_new=False)) with ( patch.object(tasks, "_get_redis", return_value=redis_client), patch.object(tasks, "_mobilede_try_recover_stalled_bootstrap_queue", return_value=False), patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[{"make_id": "3500"}]), patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")), patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config), patch.object(tasks, "_mobilede_force_full_scan_only_new", return_value=False), patch.object(tasks, "_mobilede_bootstrap_done", return_value=False), patch.object(tasks, "_mobilede_post_bootstrap_full_refresh", return_value=False), patch.object(tasks, "_mobilede_bootstrap_progress", return_value=(13, 261, 248)), patch.object(tasks, "_mobilede_bootstrap_progress_snapshot", return_value=(13, 261, 248, 2)), patch.object(tasks, "_has_recent_global_progress", return_value=False), patch.object(tasks, "_queue_mobilede_bootstrap_recovery", return_value=True) as queue_recovery, ): result = tasks.mobilede_sync_runtime_segments_task.run( lane="mobile_de_cars", delay_seconds=0.7, use_cursor=True, continuous=True, full_pass_repeat=False, ) self.assertEqual(result["status"], "bootstrap_active") queue_recovery.assert_called_once() def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None: exhausted = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=51, max_pages=50, use_cursor=False, ) not_exhausted = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=1, max_pages=50, use_cursor=False, ) cursor_window = tasks._mobilede_segment_window_is_exhausted( {"make_id": "3500", "max_pages": 50}, start_page=51, max_pages=50, use_cursor=True, ) self.assertTrue(exhausted) self.assertFalse(not_exhausted) self.assertFalse(cursor_window) def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None: segment = {"make_id": "3500", "start_page": 1} requested_start_page = 51 if segment.get("start_page") is not None and requested_start_page <= 1: effective_start_page = int(segment.get("start_page") or requested_start_page) else: effective_start_page = requested_start_page self.assertEqual(effective_start_page, 51) def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1" done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( redis_client, cycle_id="cycle-a", segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, total_segments_hint=2, ) self.assertEqual(done, 2) self.assertEqual(total, 2) self.assertTrue(should_finalize) def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None: redis_client = self._FakeRedis() redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2" redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0" done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment( redis_client, cycle_id="cycle-old", segment={"make_id": "3500", "price_min": "1", "price_max": "5000"}, total_segments_hint=2, ) self.assertEqual(done, 0) self.assertEqual(total, 0) self.assertFalse(should_finalize) def test_overflow_skip_when_known_total_fits_page_cap(self) -> None: redis_client = MagicMock() settings = MagicMock() segment = { "label": "Cars", "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000", "total_results": 995, "max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER, } added = tasks._mobilede_try_expand_overflow_segment( redis_client, settings, segment=segment, listing_count=1000, unique_count=995, max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER, segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER, ) self.assertEqual(added, 0) redis_client.sadd.assert_not_called() def test_late_overflow_children_are_not_queued_during_preplanned_bootstrap(self) -> None: redis_client = MagicMock() settings = MagicMock() segment = { "label": "Cars | ms=3500 | price=15001-20000", "search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=15001:20000", "make_id": "3500", } with patch.object(tasks, "os") as os_mock: os_mock.getenv.return_value = "true" queued = tasks._queue_mobilede_overflow_child_segments( redis_client, settings, parent_segment=segment, lane="mobile_de_cars", delay_seconds=0, use_cursor=True, only_new=False, bootstrap_run=True, refresh_cycle_id="cycle-a", ) self.assertEqual(queued, 0) redis_client.get.assert_not_called() def test_pre_split_mileage_requires_explicit_flag(self) -> None: original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False self.assertFalse( tasks._mobilede_should_pre_split_mileage( price_min=30001, price_max=50000, year_min=2010, year_max=2014, ) ) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split def test_pre_split_mileage_respects_explicit_override(self) -> None: original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True self.assertTrue( tasks._mobilede_should_pre_split_mileage( price_min=30001, price_max=50000, year_min=2010, year_max=2017, ) ) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None: import os settings = MagicMock() settings.listing.filtered_search_urls = [ "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en" ] previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") try: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true" segments = tasks._build_mobilede_runtime_segments(settings) finally: if previous is None: os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) else: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous self.assertEqual(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments)) def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None: import os settings = MagicMock() settings.listing.filtered_search_urls = [ "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en" ] previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") try: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false" segments = tasks._build_mobilede_runtime_segments(settings) finally: if previous is None: os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) else: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous self.assertGreater(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"}) self.assertTrue(all("pageNumber=1" in str(segment.get("search_url")) for segment in segments)) self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments)) def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None: import os settings = MagicMock() settings.listing.filtered_search_urls = [ "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en" ] original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START") try: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false" segments = tasks._build_mobilede_runtime_segments(settings) finally: tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split if previous_fast_start is None: os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None) else: os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start self.assertGreater(len(segments), 2) self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"}) self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments)) if __name__ == "__main__": unittest.main()