qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor worker

Browse Source
This commit is contained in:
qananasikq
2026-05-06 21:07:24 +03:00
parent 47ee4311c7
commit 81e99a41e8
6 changed files with 4302 additions and 2488 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
tests/test_worker_runtime_tasks.py
View File

@@ -1,7 +1,8 @@
from __future__ import annotations
from datetime import datetime, timezone
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from mobilede_scraper.worker import tasks
@@ -83,6 +84,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(key1, key2)
self.assertEqual(len(key1), 16)
def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
parent = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
"make_id": "3500",
"price_min": "1",
"price_max": "5000",
"year_max": "2004",
"mileage_max": "100000",
}
parent_key = tasks._mobilede_segment_fingerprint(parent)
child_a = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
"mileage_max": "50000",
"overflow_parent": parent_key,
}
child_b = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
"mileage_min": "50001",
"mileage_max": "100000",
"overflow_parent": parent_key,
}
pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
self.assertEqual(pruned, [child_a, child_b])
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
@@ -102,86 +131,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
def test_preplan_splits_dense_segment_before_queueing(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertEqual(len(planned), 2)
self.assertTrue(all(item.get("total_results") == 100 for item in planned))
self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
def test_preplan_has_hard_segment_limit(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 5000
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
{**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
{**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertGreaterEqual(len(planned), 1)
self.assertLessEqual(len(planned), 3)
def test_preplan_probe_budget_is_global(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
@@ -215,30 +164,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(calls["count"], 2)
self.assertEqual(len(planned), 5)
def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
try:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
children = tasks._mobilede_build_overflow_child_segments(
segment={
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
"price_min": "5001",
"price_max": "10000",
"year_min": "2005",
"year_max": "2009",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
},
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
finally:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
self.assertEqual(len(children), 2)
self.assertEqual(children[0].get("mileage_max"), "150000")
self.assertEqual(children[1].get("mileage_min"), "150001")
self.assertIsNone(children[1].get("mileage_max"))
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
@@ -278,6 +203,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
redis_client = self._FakeRedis()
started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
persistence = MagicMock()
persistence.mark_sold_not_seen_since.return_value = 42
with (
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
):
finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
redis_client,
cycle_id="cycle-lag",
total_segments_hint=377,
)
self.assertTrue(finalized)
self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
persistence.mark_sold_not_seen_since.assert_called_once_with(
started_at,
prefix=("mobile.de:", "mobilede:"),
safety_ratio=0.8,
)
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
@@ -357,9 +310,32 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(done, 0)
self.assertEqual(total, 2)
self.assertEqual(total, 0)
self.assertFalse(should_finalize)
def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
segment = {
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
"total_results": 995,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
added = tasks._mobilede_try_expand_overflow_segment(
redis_client,
settings,
segment=segment,
listing_count=1000,
unique_count=995,
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
self.assertEqual(added, 0)
redis_client.sadd.assert_not_called()
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
@@ -390,58 +366,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
search_url = (
"https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
"?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
)
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
segments = tasks._expand_mobilede_search_url_segment(
{
"label": "Cars",
"search_url": search_url,
"start_page": 1,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
make_counts: dict[str, int] = {}
for segment in segments:
make_id = str(segment.get("make_id") or "")
make_counts[make_id] = make_counts.get(make_id, 0) + 1
self.assertIsNone(segment.get("mileage_min"))
self.assertIsNone(segment.get("mileage_max"))
self.assertEqual(make_counts, {"3500": 59, "11000": 59})
self.assertEqual(len(segments), 118)
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
]
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertEqual(len(segments), 118)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
import os