qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refresh api and tests

Browse Source
This commit is contained in:
qananasikq
2026-05-11 12:46:07 +03:00
parent d9d073641b
commit 2cebb78316
12 changed files with 621 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
tests/test_worker_runtime_tasks.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
import unittest
from unittest.mock import MagicMock, patch
@@ -12,6 +13,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
def __init__(self) -> None:
self.store: dict[str, str] = {}
self.sets: dict[str, set[str]] = {}
self.lists: dict[str, list[str]] = {}
def get(self, key: str):
return self.store.get(key)
@@ -34,6 +36,26 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
bucket.add(member)
return 1 if len(bucket) > before else 0
def scard(self, key: str):
return len(self.sets.get(key, set()))
def llen(self, key: str):
return len(self.lists.get(key, []))
def delete(self, *keys: str):
removed = 0
for key in keys:
if key in self.store:
del self.store[key]
removed += 1
if key in self.sets:
del self.sets[key]
removed += 1
if key in self.lists:
del self.lists[key]
removed += 1
return removed
def expire(self, key: str, ttl: int): # noqa: ARG002
return True
@@ -203,6 +225,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
effective_only_new = tasks._mobilede_force_full_scan_only_new(
True,
redis_client=redis_client,
continuous=True,
)
self.assertFalse(effective_only_new)
def test_post_bootstrap_refresh_disabled_for_continuous_hourly_cycle(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
post_bootstrap_refresh = tasks._mobilede_post_bootstrap_full_refresh(
redis_client,
False,
continuous=True,
)
self.assertFalse(post_bootstrap_refresh)
def test_stalled_bootstrap_recovery_ignores_stale_global_progress_key(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "1"
redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
self.assertTrue(recovered)
self.assertNotIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
def test_stalled_bootstrap_recovery_keeps_recent_progress(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "9950"
redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
self.assertFalse(recovered)
self.assertIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
redis_client = self._FakeRedis()
started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
@@ -244,6 +318,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(should_skip)
def test_queue_mobilede_bootstrap_recovery_can_force_requeue_existing_pending(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
with patch.object(tasks.mobilede_sync_runtime_segments_task, "apply_async") as apply_async:
queued = tasks._queue_mobilede_bootstrap_recovery(
redis_client,
lane="mobile_de_cars",
delay_seconds=0.7,
use_cursor=True,
continuous=True,
segment_label="runtime_segments",
reason="waiting_for_active_bootstrap_tasks",
countdown=5,
force=True,
)
self.assertTrue(queued)
apply_async.assert_called_once()
self.assertEqual(redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY], "1")
def test_runtime_segments_task_requeues_pending_bootstrap_recovery_while_tasks_are_still_active(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
runtime_config = SimpleNamespace(sync=SimpleNamespace(only_new=False))
with (
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_mobilede_try_recover_stalled_bootstrap_queue", return_value=False),
patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[{"make_id": "3500"}]),
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
patch.object(tasks, "_mobilede_force_full_scan_only_new", return_value=False),
patch.object(tasks, "_mobilede_bootstrap_done", return_value=False),
patch.object(tasks, "_mobilede_post_bootstrap_full_refresh", return_value=False),
patch.object(tasks, "_mobilede_bootstrap_progress", return_value=(13, 261, 248)),
patch.object(tasks, "_mobilede_bootstrap_progress_snapshot", return_value=(13, 261, 248, 2)),
patch.object(tasks, "_has_recent_global_progress", return_value=False),
patch.object(tasks, "_queue_mobilede_bootstrap_recovery", return_value=True) as queue_recovery,
):
result = tasks.mobilede_sync_runtime_segments_task.run(
lane="mobile_de_cars",
delay_seconds=0.7,
use_cursor=True,
continuous=True,
full_pass_repeat=False,
)
self.assertEqual(result["status"], "bootstrap_active")
queue_recovery.assert_called_once()
def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
exhausted = tasks._mobilede_segment_window_is_exhausted(
{"make_id": "3500", "max_pages": 50},
@@ -336,6 +462,32 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(added, 0)
redis_client.sadd.assert_not_called()
def test_late_overflow_children_are_not_queued_during_preplanned_bootstrap(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
segment = {
"label": "Cars | ms=3500 | price=15001-20000",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=15001:20000",
"make_id": "3500",
}
with patch.object(tasks, "os") as os_mock:
os_mock.getenv.return_value = "true"
queued = tasks._queue_mobilede_overflow_child_segments(
redis_client,
settings,
parent_segment=segment,
lane="mobile_de_cars",
delay_seconds=0,
use_cursor=True,
only_new=False,
bootstrap_run=True,
refresh_cycle_id="cycle-a",
)
self.assertEqual(queued, 0)
redis_client.get.assert_not_called()
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
@@ -390,6 +542,37 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
]
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("pageNumber=1" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222"
]
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
self.assertGreater(len(segments), 300)
self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments))
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))
if __name__ == "__main__":
unittest.main()