396 lines
15 KiB
Python
396 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from mobilede_scraper.worker import tasks
|
|
|
|
|
|
class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
|
|
class _FakeRedis:
|
|
def __init__(self) -> None:
|
|
self.store: dict[str, str] = {}
|
|
self.sets: dict[str, set[str]] = {}
|
|
|
|
def get(self, key: str):
|
|
return self.store.get(key)
|
|
|
|
def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002
|
|
if nx and key in self.store:
|
|
return False
|
|
self.store[key] = str(value)
|
|
return True
|
|
|
|
def incr(self, key: str):
|
|
current = int(self.store.get(key, "0"))
|
|
current += 1
|
|
self.store[key] = str(current)
|
|
return current
|
|
|
|
def sadd(self, key: str, member: str):
|
|
bucket = self.sets.setdefault(key, set())
|
|
before = len(bucket)
|
|
bucket.add(member)
|
|
return 1 if len(bucket) > before else 0
|
|
|
|
def expire(self, key: str, ttl: int): # noqa: ARG002
|
|
return True
|
|
|
|
def test_lock_acquire_refresh_release(self) -> None:
|
|
redis_client = MagicMock()
|
|
|
|
redis_client.set.return_value = True
|
|
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
|
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
|
|
|
redis_client.eval.return_value = 1
|
|
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
|
|
|
redis_client.eval.reset_mock()
|
|
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
|
args = redis_client.eval.call_args[0]
|
|
self.assertEqual(args[2], "lock:key")
|
|
self.assertEqual(args[3], "owner-token")
|
|
|
|
def test_mobilede_segment_key_is_stable(self) -> None:
|
|
segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000}
|
|
|
|
key1 = tasks._mobilede_task_segment_key(
|
|
segment=segment,
|
|
search_url=None,
|
|
make_id=None,
|
|
model_id=None,
|
|
price_min=None,
|
|
price_max=None,
|
|
year_min=None,
|
|
year_max=None,
|
|
mileage_min=None,
|
|
mileage_max=None,
|
|
)
|
|
key2 = tasks._mobilede_task_segment_key(
|
|
segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000},
|
|
search_url=None,
|
|
make_id=None,
|
|
model_id=None,
|
|
price_min=None,
|
|
price_max=None,
|
|
year_min=None,
|
|
year_max=None,
|
|
mileage_min=None,
|
|
mileage_max=None,
|
|
)
|
|
|
|
self.assertEqual(key1, key2)
|
|
self.assertEqual(len(key1), 16)
|
|
|
|
def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
|
|
parent = {
|
|
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
|
|
"make_id": "3500",
|
|
"price_min": "1",
|
|
"price_max": "5000",
|
|
"year_max": "2004",
|
|
"mileage_max": "100000",
|
|
}
|
|
parent_key = tasks._mobilede_segment_fingerprint(parent)
|
|
child_a = {
|
|
**parent,
|
|
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
|
|
"mileage_max": "50000",
|
|
"overflow_parent": parent_key,
|
|
}
|
|
child_b = {
|
|
**parent,
|
|
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
|
|
"mileage_min": "50001",
|
|
"mileage_max": "100000",
|
|
"overflow_parent": parent_key,
|
|
}
|
|
|
|
pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
|
|
|
|
self.assertEqual(pruned, [child_a, child_b])
|
|
|
|
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
|
|
redis_client = MagicMock()
|
|
settings = MagicMock()
|
|
redis_client.get.side_effect = lambda key: {
|
|
tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]',
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2",
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2",
|
|
}.get(key)
|
|
redis_client.incr.return_value = 2
|
|
redis_client.ttl.return_value = -2
|
|
|
|
segment = tasks._reserve_mobilede_runtime_segment(
|
|
redis_client,
|
|
settings,
|
|
only_new=False,
|
|
)
|
|
|
|
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
|
|
|
|
def test_preplan_probe_budget_is_global(self) -> None:
|
|
original_probe = tasks._mobilede_probe_segment_total
|
|
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
|
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
|
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
|
|
|
calls = {"count": 0}
|
|
segments = [
|
|
{"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"}
|
|
for index in range(5)
|
|
]
|
|
|
|
try:
|
|
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
|
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50
|
|
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2
|
|
|
|
def _probe(_segment):
|
|
calls["count"] += 1
|
|
return 100
|
|
|
|
tasks._mobilede_probe_segment_total = _probe
|
|
|
|
planned = tasks._mobilede_preplan_runtime_segments(segments)
|
|
finally:
|
|
tasks._mobilede_probe_segment_total = original_probe
|
|
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
|
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
|
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
|
|
|
self.assertEqual(calls["count"], 2)
|
|
self.assertEqual(len(planned), 5)
|
|
|
|
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
|
|
redis_client = MagicMock()
|
|
redis_client.get.side_effect = lambda key: {
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
|
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
|
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
|
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
|
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
|
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
|
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
|
}.get(key)
|
|
redis_client.scard.return_value = 1
|
|
|
|
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
|
|
|
self.assertFalse(finalized)
|
|
redis_client.set.assert_not_called()
|
|
|
|
def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None:
|
|
redis_client = MagicMock()
|
|
redis_client.get.side_effect = lambda key: {
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
|
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
|
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
|
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
|
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
|
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
|
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
|
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
|
}.get(key)
|
|
redis_client.scard.return_value = 0
|
|
|
|
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
|
|
|
self.assertTrue(finalized)
|
|
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
|
|
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
|
|
|
|
def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
|
|
redis_client = self._FakeRedis()
|
|
started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
|
|
persistence = MagicMock()
|
|
persistence.mark_sold_not_seen_since.return_value = 42
|
|
|
|
with (
|
|
patch.object(tasks, "_get_persistence", return_value=persistence),
|
|
patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
|
|
):
|
|
finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
|
|
redis_client,
|
|
cycle_id="cycle-lag",
|
|
total_segments_hint=377,
|
|
)
|
|
|
|
self.assertTrue(finalized)
|
|
self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
|
|
persistence.mark_sold_not_seen_since.assert_called_once_with(
|
|
started_at,
|
|
prefix=("mobile.de:", "mobilede:"),
|
|
safety_ratio=0.8,
|
|
)
|
|
|
|
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
|
|
redis_client = MagicMock()
|
|
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
|
|
|
|
should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment(
|
|
redis_client,
|
|
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
|
only_new=False,
|
|
bootstrap_run_active=True,
|
|
)
|
|
|
|
self.assertTrue(should_skip)
|
|
|
|
def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
|
|
exhausted = tasks._mobilede_segment_window_is_exhausted(
|
|
{"make_id": "3500", "max_pages": 50},
|
|
start_page=51,
|
|
max_pages=50,
|
|
use_cursor=False,
|
|
)
|
|
not_exhausted = tasks._mobilede_segment_window_is_exhausted(
|
|
{"make_id": "3500", "max_pages": 50},
|
|
start_page=1,
|
|
max_pages=50,
|
|
use_cursor=False,
|
|
)
|
|
cursor_window = tasks._mobilede_segment_window_is_exhausted(
|
|
{"make_id": "3500", "max_pages": 50},
|
|
start_page=51,
|
|
max_pages=50,
|
|
use_cursor=True,
|
|
)
|
|
|
|
self.assertTrue(exhausted)
|
|
self.assertFalse(not_exhausted)
|
|
self.assertFalse(cursor_window)
|
|
|
|
def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None:
|
|
segment = {"make_id": "3500", "start_page": 1}
|
|
|
|
requested_start_page = 51
|
|
if segment.get("start_page") is not None and requested_start_page <= 1:
|
|
effective_start_page = int(segment.get("start_page") or requested_start_page)
|
|
else:
|
|
effective_start_page = requested_start_page
|
|
|
|
self.assertEqual(effective_start_page, 51)
|
|
|
|
def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None:
|
|
redis_client = self._FakeRedis()
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1"
|
|
|
|
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
|
redis_client,
|
|
cycle_id="cycle-a",
|
|
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
|
total_segments_hint=2,
|
|
)
|
|
|
|
self.assertEqual(done, 2)
|
|
self.assertEqual(total, 2)
|
|
self.assertTrue(should_finalize)
|
|
|
|
def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None:
|
|
redis_client = self._FakeRedis()
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
|
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0"
|
|
|
|
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
|
redis_client,
|
|
cycle_id="cycle-old",
|
|
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
|
total_segments_hint=2,
|
|
)
|
|
|
|
self.assertEqual(done, 0)
|
|
self.assertEqual(total, 0)
|
|
self.assertFalse(should_finalize)
|
|
|
|
def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
|
|
redis_client = MagicMock()
|
|
settings = MagicMock()
|
|
segment = {
|
|
"label": "Cars",
|
|
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
|
|
"total_results": 995,
|
|
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
|
}
|
|
|
|
added = tasks._mobilede_try_expand_overflow_segment(
|
|
redis_client,
|
|
settings,
|
|
segment=segment,
|
|
listing_count=1000,
|
|
unique_count=995,
|
|
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
|
segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
|
)
|
|
|
|
self.assertEqual(added, 0)
|
|
redis_client.sadd.assert_not_called()
|
|
|
|
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
|
|
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
|
try:
|
|
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
|
|
self.assertFalse(
|
|
tasks._mobilede_should_pre_split_mileage(
|
|
price_min=30001,
|
|
price_max=50000,
|
|
year_min=2010,
|
|
year_max=2014,
|
|
)
|
|
)
|
|
finally:
|
|
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
|
|
|
def test_pre_split_mileage_respects_explicit_override(self) -> None:
|
|
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
|
try:
|
|
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
|
|
self.assertTrue(
|
|
tasks._mobilede_should_pre_split_mileage(
|
|
price_min=30001,
|
|
price_max=50000,
|
|
year_min=2010,
|
|
year_max=2017,
|
|
)
|
|
)
|
|
finally:
|
|
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
|
|
|
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
|
|
import os
|
|
|
|
settings = MagicMock()
|
|
settings.listing.filtered_search_urls = [
|
|
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
|
|
]
|
|
|
|
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
|
|
try:
|
|
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true"
|
|
segments = tasks._build_mobilede_runtime_segments(settings)
|
|
finally:
|
|
if previous is None:
|
|
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
|
|
else:
|
|
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
|
|
|
|
self.assertEqual(len(segments), 2)
|
|
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
|
|
self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments))
|
|
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
|
|
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|