qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build complete runtime segments

Browse Source
This commit is contained in:
qananasikq
2026-08-19 17:11:00 +03:00
parent 8297742a64
commit 7068b7e8fa
5 changed files with 1054 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_worker_runtime_tasks.py
+602 -52
View File
@@ -1,14 +1,40 @@
from __future__ import annotations
from datetime import datetime, timezone
import json
from types import SimpleNamespace
import unittest
from unittest.mock import MagicMock, patch
from mobilede_scraper.worker import planner, tasks
from mobilede_scraper.core.runtime_config import RuntimeConfig
from mobilede_scraper.worker import planner, refresh_cycle, search_sync, tasks
class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
class _FakeDetailPersistence:
def __init__(self) -> None:
self.enqueued: list[str] = []
self.released: list[tuple[str, str | None]] = []
def enqueue_detail_jobs(self, origin_ids, *, priority=3, reset=False): # noqa: ARG002
self.enqueued = list(origin_ids)
return {"ready": len(self.enqueued), "deferred": 0, "missing": 0}
def reserve_detail_jobs(self, *, limit, lease_seconds=900, listing_ids=None): # noqa: ARG002
return [
{
"listing_id": listing_id,
"lease_owner": f"lease-{listing_id}",
"attempts": 1,
"priority": 9,
}
for listing_id in list(listing_ids or [])[:limit]
]
def release_detail_job(self, listing_id, *, lease_owner=None):
self.released.append((listing_id, lease_owner))
return True
class _FakeRedis:
def __init__(self) -> None:
self.store: dict[str, str] = {}
@@ -36,6 +62,13 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
bucket.add(member)
return 1 if len(bucket) > before else 0
def srem(self, key: str, member: str):
bucket = self.sets.setdefault(key, set())
if member not in bucket:
return 0
bucket.remove(member)
return 1
def scard(self, key: str):
return len(self.sets.get(key, set()))
@@ -78,6 +111,33 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(args[2], "lock:key")
self.assertEqual(args[3], "owner-token")
def test_refresh_finalization_claim_is_released_after_db_failure(self) -> None:
redis_client = self._FakeRedis()
cycle_id = "cycle-1"
redis_client.set(tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id)
redis_client.set(tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY, "1")
redis_client.set(tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, "1")
redis_client.set(tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, datetime.now(timezone.utc).isoformat())
self.assertTrue(refresh_cycle._mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id))
persistence = MagicMock()
persistence.mark_sold_not_seen_since.side_effect = RuntimeError("db unavailable")
with self.assertRaisesRegex(RuntimeError, "db unavailable"):
refresh_cycle._mobilede_finalize_refresh_cycle_sold_marking(
redis_client,
cycle_id=cycle_id,
logger=MagicMock(),
get_persistence=lambda: persistence,
origin_prefixes=("mobile.de:",),
post_refresh_probe_enabled=False,
post_refresh_probe_batch_size=0,
schedule_post_refresh_probe=lambda: None,
)
claim_key = refresh_cycle._mobilede_refresh_cycle_finalized_key(cycle_id)
self.assertIsNone(redis_client.get(claim_key))
self.assertTrue(refresh_cycle._mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id))
def test_mobilede_segment_key_is_stable(self) -> None:
segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000}
@@ -109,6 +169,178 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(key1, key2)
self.assertEqual(len(key1), 16)
def test_runtime_numeric_filters_intersect_planner_segment(self) -> None:
self.assertEqual(
search_sync._intersect_numeric_bounds("10000", "30000", 15000, 25000),
("15000", "25000", True),
)
self.assertEqual(
search_sync._intersect_numeric_bounds("10000", "12000", 15000, 25000),
("10000", "12000", False),
)
def test_depth_six_overflow_leaf_can_split_further(self) -> None:
segment = {
"search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ml=%3A4687&p=15003%3A",
"listing_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ml=%3A4687&p=15003%3A",
"label": "BMW hot leaf",
"make_id": "3500",
"mileage_max": "4687",
"price_min": "15003",
"overflow_depth": 6,
"overflow_split": "mileage",
"max_pages": 50,
}
with patch.object(planner, "MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", 8):
groups = planner._mobilede_build_overflow_candidate_groups(segment=segment, max_pages=50)
self.assertTrue(groups)
self.assertTrue(all(int(child["overflow_depth"]) == 7 for _kind, children in groups for child in children))
self.assertTrue(any(kind == "price" for kind, _children in groups))
def test_detail_queue_prioritizes_new_unique_listings(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true", "MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "3"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
redis_client,
["mobile.de:100", "mobile.de:100", "mobile.de:101"],
priority=9,
)
self.assertEqual(result, {"queued": 2, "duplicate": 0, "full": 0})
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"100", "101"})
self.assertEqual(apply_async.call_count, 2)
self.assertTrue(all(call.kwargs["queue"] == tasks.MOBILEDE_IMAGES_QUEUE for call in apply_async.call_args_list))
self.assertTrue(all(call.kwargs["priority"] == 9 for call in apply_async.call_args_list))
self.assertEqual(
[call.kwargs["kwargs"] for call in apply_async.call_args_list],
[
{"listing_id": "100", "lane": "mobile_de_cars", "lease_owner": "lease-100", "attempt": 1},
{"listing_id": "101", "lane": "mobile_de_cars", "lease_owner": "lease-101", "attempt": 1},
],
)
def test_detail_queue_respects_pending_limit(self) -> None:
redis_client = self._FakeRedis()
redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY] = {"100", "101"}
persistence = self._FakeDetailPersistence()
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true", "MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "2"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
redis_client,
["mobile.de:102", "mobile.de:103"],
)
self.assertEqual(result, {"queued": 0, "duplicate": 0, "full": 2})
apply_async.assert_not_called()
def test_detail_queue_skips_recently_completed_listing(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
completed_key = tasks.MOBILEDE_DETAIL_COMPLETED_KEY_FMT.format(listing_id="100")
redis_client.store[completed_key] = "1"
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
redis_client,
["mobile.de:100", "mobile.de:101"],
)
self.assertEqual(result, {"queued": 1, "duplicate": 1, "full": 0})
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"101"})
apply_async.assert_called_once()
def test_explicit_gallery_refresh_invalidates_completed_listing(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
completed_key = tasks.MOBILEDE_DETAIL_COMPLETED_KEY_FMT.format(listing_id="100")
redis_client.store[completed_key] = "1"
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
redis_client,
["mobile.de:100"],
invalidate_completed=True,
)
self.assertEqual(result, {"queued": 1, "duplicate": 0, "full": 0})
self.assertNotIn(completed_key, redis_client.store)
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"100"})
apply_async.assert_called_once()
def test_detail_queue_releases_lease_when_dispatch_fails(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async", side_effect=RuntimeError("broker down")),
self.assertRaisesRegex(RuntimeError, "broker down"),
):
tasks._queue_mobilede_detail_listing_ids(redis_client, ["mobile.de:100"])
self.assertEqual(persistence.released, [("100", "lease-100")])
self.assertNotIn("100", redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY])
def test_only_new_segment_with_updates_only_enters_cooldown(self) -> None:
redis_client = self._FakeRedis()
segment = {"make_id": "BMW", "label": "BMW"}
tasks._mobilede_update_segment_freshness_state(
redis_client,
segment=segment,
only_new=True,
inserted=0,
updated=20,
listings=20,
)
cooldown_key = tasks._mobilede_segment_cooldown_key(segment)
hot_key = tasks._mobilede_segment_hot_key(segment)
self.assertEqual(redis_client.store[cooldown_key], "1")
self.assertNotIn(hot_key, redis_client.store)
def test_detail_scraper_is_reused_within_worker_process(self) -> None:
original = tasks._detail_scraper_instance
tasks._detail_scraper_instance = None
try:
with (
patch.object(tasks.MobileDeClient, "for_worker") as client_factory,
patch.object(tasks, "_get_persistence") as persistence_factory,
patch.object(tasks, "MobileDeScraper") as scraper_factory,
):
expected = scraper_factory.return_value
first = tasks._get_detail_scraper()
second = tasks._get_detail_scraper()
self.assertIs(first, expected)
self.assertIs(second, expected)
client_factory.assert_called_once_with(delay_seconds=0)
persistence_factory.assert_called_once_with()
scraper_factory.assert_called_once()
finally:
tasks._detail_scraper_instance = original
def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
parent = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
@@ -137,6 +369,256 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(pruned, [child_a, child_b])
def test_compact_runtime_segments_prunes_observed_empty_and_merges_adjacent_ranges(self) -> None:
base_url = "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&fr=2018:2020"
segments = [
{
"label": "BMW | price=1-100 | year=2018-2020 | total=400",
"search_url": f"{base_url}&p=1:100",
"source_search_url": base_url,
"make_id": "3500",
"price_min": "1",
"price_max": "100",
"year_min": "2018",
"year_max": "2020",
"total_results": 400,
"observed_complete": True,
"max_pages": 50,
},
{
"label": "BMW | price=101-200 | year=2018-2020 | total=500",
"search_url": f"{base_url}&p=101:200",
"source_search_url": base_url,
"make_id": "3500",
"price_min": "101",
"price_max": "200",
"year_min": "2018",
"year_max": "2020",
"total_results": 500,
"observed_complete": True,
"max_pages": 50,
},
{
"label": "BMW | price=201-300 | year=2018-2020 | total=0",
"search_url": f"{base_url}&p=201:300",
"source_search_url": base_url,
"make_id": "3500",
"price_min": "201",
"price_max": "300",
"year_min": "2018",
"year_max": "2020",
"total_results": 0,
"observed_complete": True,
"max_pages": 50,
},
]
compacted = tasks._mobilede_compact_runtime_segments(segments)
self.assertEqual(len(compacted), 1)
self.assertEqual(compacted[0]["price_min"], "1")
self.assertEqual(compacted[0]["price_max"], "200")
self.assertEqual(compacted[0]["total_results"], 900)
self.assertEqual(compacted[0]["max_pages"], 45)
self.assertEqual(compacted[0]["compacted_from"], 2)
def test_finalize_runtime_plan_compacts_before_interleaving_makes(self) -> None:
base_url = "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car"
segments = [
{
"label": "BMW | price=1-100 | total=400",
"search_url": f"{base_url}&ms=3500&p=1:100",
"source_search_url": base_url,
"make_id": "3500",
"price_min": "1",
"price_max": "100",
"total_results": 400,
"observed_complete": True,
"max_pages": 20,
},
{
"label": "BMW | price=101-200 | total=500",
"search_url": f"{base_url}&ms=3500&p=101:200",
"source_search_url": base_url,
"make_id": "3500",
"price_min": "101",
"price_max": "200",
"total_results": 500,
"observed_complete": True,
"max_pages": 25,
},
{
"label": "Porsche | price=1-5000 | total=100",
"search_url": f"{base_url}&ms=20100&p=1:5000",
"source_search_url": base_url,
"make_id": "20100",
"price_min": "1",
"price_max": "5000",
"total_results": 100,
"observed_complete": True,
"max_pages": 5,
},
]
finalized = planner._mobilede_finalize_runtime_plan(segments)
self.assertEqual(len(finalized), 2)
self.assertEqual(finalized[0]["make_id"], "3500")
self.assertEqual(finalized[0]["total_results"], 900)
self.assertEqual(finalized[1]["make_id"], "20100")
def test_complete_runtime_plan_rejects_unknown_and_dense_leaves(self) -> None:
with self.assertRaisesRegex(RuntimeError, "unknown=1 dense=1"):
planner._mobilede_assert_complete_runtime_plan(
[
{"label": "unknown", "total_results": None},
{"label": "dense", "total_results": 1001},
{"label": "valid", "total_results": 950},
]
)
def test_complete_dense_refine_ignores_soft_probe_limit(self) -> None:
parent = {
"label": "BMW dense",
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:3000",
"make_id": "3500",
"price_min": "1",
"price_max": "3000",
"total_results": 2400,
"max_pages": 50,
}
parent_key = tasks._mobilede_segment_fingerprint(parent)
children = [
{
**parent,
"label": f"BMW child {index}",
"search_url": f"https://www.mobile.de/ru/search.html?ms=3500&p={start}:{end}",
"price_min": str(start),
"price_max": str(end),
"total_results": None,
"overflow_parent": parent_key,
"overflow_depth": 1,
}
for index, (start, end) in enumerate(((1, 1000), (1001, 2000), (2001, 3000)), start=1)
]
with (
patch.object(planner, "_mobilede_build_overflow_child_segments", return_value=children),
patch.object(planner, "_mobilede_probe_segment_total", side_effect=[800, 800, 800]) as probe,
):
refined = planner._mobilede_refine_dense_planned_segments([parent])
self.assertEqual(probe.call_count, 3)
self.assertEqual([item["total_results"] for item in refined], [800, 800, 800])
planner._mobilede_assert_complete_runtime_plan(refined)
def test_dense_refine_recursively_completes_tree_in_one_call(self) -> None:
parent = {
"label": "BMW parent",
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:3000",
"make_id": "3500",
"price_min": "1",
"price_max": "3000",
"total_results": 2400,
"max_pages": 50,
}
child_dense = {
**parent,
"label": "BMW dense child",
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:2000",
"price_max": "2000",
"total_results": None,
"overflow_depth": 1,
}
child_complete = {
**parent,
"label": "BMW complete child",
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=2001:3000",
"price_min": "2001",
"total_results": None,
"overflow_depth": 1,
}
grandchildren = [
{
**child_dense,
"label": f"BMW grandchild {index}",
"search_url": f"https://www.mobile.de/ru/search.html?ms=3500&p={start}:{end}",
"price_min": str(start),
"price_max": str(end),
"total_results": None,
"overflow_depth": 2,
}
for index, (start, end) in enumerate(((1, 1000), (1001, 2000)), start=1)
]
with (
patch.object(
planner,
"_mobilede_build_overflow_child_segments",
side_effect=[[child_dense, child_complete], grandchildren],
) as split,
patch.object(
planner,
"_mobilede_probe_segment_total",
side_effect=[1500, 900, 700, 800],
),
):
refined = planner._mobilede_refine_dense_planned_segments([parent])
self.assertEqual(split.call_count, 2)
self.assertEqual(sorted(item["total_results"] for item in refined), [700, 800, 900])
planner._mobilede_assert_complete_runtime_plan(refined)
def test_record_observation_keeps_active_bootstrap_cache_shape(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
first = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:100",
"source_search_url": "https://www.mobile.de/ru/search.html?ms=3500",
"make_id": "3500",
"price_min": "1",
"price_max": "100",
"total_results": None,
"max_pages": 50,
}
second = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=101:200",
"source_search_url": "https://www.mobile.de/ru/search.html?ms=3500",
"make_id": "3500",
"price_min": "101",
"price_max": "200",
"total_results": 500,
"observed_complete": True,
"max_pages": 25,
}
with (
patch.object(tasks, "_acquire_lock", return_value=True),
patch.object(tasks, "_release_lock_if_owner"),
patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[first, second]),
patch.object(planner, "_mobilede_source_segments_from_settings", return_value=[first]),
patch.object(planner, "_mobilede_save_learned_runtime_segments") as save_plan,
):
removed = tasks._mobilede_record_observed_segment_total(
redis_client,
settings,
segment=first,
total_results=400,
listing_count=400,
unique_count=400,
bootstrap_run=True,
)
cached_payload = redis_client.set.call_args.args[1]
cached_segments = json.loads(cached_payload)
self.assertEqual(removed, 1)
self.assertEqual(len(cached_segments), 2)
self.assertEqual(cached_segments[0]["total_results"], 400)
self.assertTrue(cached_segments[0]["observed_complete"])
learned_segments = save_plan.call_args.args[1]
self.assertEqual(len(learned_segments), 1)
self.assertEqual(learned_segments[0]["total_results"], 900)
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
@@ -319,6 +801,57 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(sold_marked, 0)
persistence.mark_sold_not_seen_since.assert_not_called()
def test_known_complete_filtered_scope_is_safe_for_reconciliation(self) -> None:
complete_url = "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=25100&ms=8600&ms=15200&ms=3500&ref=dsp"
runtime_config = RuntimeConfig()
settings = MagicMock()
settings.listing.filtered_search_urls = [complete_url]
with patch.dict(
"os.environ",
{"MOBILEDE_COMPLETE_SCOPE_BRANDS": "BMW,Volvo,Ferrari,Lexus"},
clear=False,
):
reasons = tasks._mobilede_complete_scope_unsafe_reasons(runtime_config, settings)
self.assertEqual(reasons, [])
def test_partial_filtered_scope_remains_unsafe_for_reconciliation(self) -> None:
runtime_config = RuntimeConfig()
runtime_config.sync.limit = 100
settings = MagicMock()
settings.listing.filtered_search_urls = ["https://www.mobile.de/ru/search.html?ms=3500"]
with patch.dict(
"os.environ",
{
"MOBILEDE_COMPLETE_SCOPE_BRANDS": "BMW,Volvo,Ferrari,Lexus",
},
clear=False,
):
reasons = tasks._mobilede_complete_scope_unsafe_reasons(runtime_config, settings)
self.assertIn("sync_limit", reasons)
def test_effective_full_pass_is_safe_after_only_new_bootstrap(self) -> None:
runtime_config = RuntimeConfig()
runtime_config.sync.only_new = True
settings = MagicMock()
settings.listing.filtered_search_urls = ["https://www.mobile.de/ru/search.html?ms=25100&ms=8600&ms=15200&ms=3500"]
with patch.dict(
"os.environ",
{"MOBILEDE_COMPLETE_SCOPE_BRANDS": "BMW,Volvo,Ferrari,Lexus"},
clear=False,
):
reasons = tasks._mobilede_complete_scope_unsafe_reasons(
runtime_config,
settings,
effective_only_new=False,
)
self.assertEqual(reasons, [])
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
@@ -485,7 +1018,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
"make_id": "3500",
}
with patch.object(tasks, "os") as os_mock:
with patch.object(planner, "os") as os_mock:
os_mock.getenv.return_value = "true"
queued = tasks._queue_mobilede_overflow_child_segments(
redis_client,
@@ -502,6 +1035,55 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(queued, 0)
redis_client.get.assert_not_called()
def test_late_overflow_split_is_saved_for_next_pass_without_extending_bootstrap(self) -> None:
redis_client = MagicMock()
redis_client.sadd.return_value = 1
settings = MagicMock()
parent = {
"label": "Cars | ms=3500 | price=1-5000",
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000",
"make_id": "3500",
"price_min": "1",
"price_max": "5000",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
parent_key = tasks._mobilede_segment_fingerprint(parent)
child = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:2500",
"price_max": "2500",
"overflow_parent": parent_key,
"overflow_depth": 1,
}
with (
patch.object(planner, "_mobilede_build_overflow_child_segments", return_value=[child]),
patch.object(tasks, "_acquire_lock", return_value=True),
patch.object(tasks, "_release_lock_if_owner"),
patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[parent]),
patch.object(planner, "_mobilede_source_segments_from_settings", return_value=[parent]),
patch.object(planner, "_mobilede_save_learned_runtime_segments") as save_plan,
patch.object(planner, "_mobilede_skip_late_overflow_children_during_bootstrap", return_value=True),
):
added = tasks._mobilede_try_expand_overflow_segment(
redis_client,
settings,
segment=parent,
listing_count=1000,
unique_count=1000,
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
bootstrap_run=True,
)
self.assertEqual(added, 0)
self.assertFalse(
any(call.args and call.args[0] == tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY for call in redis_client.set.call_args_list)
)
save_plan.assert_called_once()
learned_segments = save_plan.call_args.args[1]
self.assertEqual(learned_segments, [child])
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
@@ -532,53 +1114,24 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true"
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
if previous is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
self.assertEqual(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&lang=en"
]
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
with (
patch.object(planner, "_mobilede_try_keep_root_segment_unsplit", return_value=None),
patch.object(planner, "_expand_mobilede_search_url_segment_by_probe", return_value=None),
patch.object(planner, "_mobilede_probe_total", return_value=None),
patch.object(planner, "_mobilede_preplan_runtime_segments", side_effect=lambda items: items),
):
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
if previous is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
with (
patch.object(planner, "_mobilede_try_keep_root_segment_unsplit", return_value=None),
patch.object(planner, "_expand_mobilede_search_url_segment_by_probe", return_value=None),
patch.object(planner, "_mobilede_probe_total", return_value=None),
patch.object(
planner,
"_mobilede_preplan_runtime_segments",
side_effect=lambda items: [{**item, "total_results": 1} for item in items],
),
patch.object(planner, "_mobilede_compact_runtime_segments", side_effect=lambda items: items),
):
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
@@ -586,31 +1139,28 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
import os
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222&lang=en"
]
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
previous_fast_start = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
with (
patch.object(planner, "_mobilede_try_keep_root_segment_unsplit", return_value=None),
patch.object(planner, "_expand_mobilede_search_url_segment_by_probe", return_value=None),
patch.object(planner, "_mobilede_probe_total", return_value=None),
patch.object(planner, "_mobilede_preplan_runtime_segments", side_effect=lambda items: items),
patch.object(
planner,
"_mobilede_preplan_runtime_segments",
side_effect=lambda items: [{**item, "total_results": 1} for item in items],
),
patch.object(planner, "_mobilede_compact_runtime_segments", side_effect=lambda items: items),
):
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
if previous_fast_start is None:
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
else:
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous_fast_start
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})