qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor worker

Browse Source
This commit is contained in:
qananasikq
2026-05-06 21:07:24 +03:00
parent 47ee4311c7
commit 81e99a41e8
6 changed files with 4302 additions and 2488 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
mobilede_scraper/worker/constants.py
View File

@@ -36,9 +36,13 @@ MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
)
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180"))))
MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60))))
MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
@@ -80,6 +84,10 @@ MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200")))
MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5"))))
MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
@@ -98,6 +106,35 @@ MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
1,
min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
)
MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
1,
min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))),
)
MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max(
1,
min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))),
)
MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min(
1.0,
max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))),
)
MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min(
1.2,
max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))),
)
MOBILEDE_SEGMENT_TINY_RATIO = min(
0.8,
max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))),
)
MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min(
1.0,
max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))),
)
MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
0,
min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))),
)
MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"

2145
mobilede_scraper/worker/planner.py Normal file
View File

File diff suppressed because it is too large Load Diff

260
mobilede_scraper/worker/refresh_cycle.py Normal file
View File

@@ -0,0 +1,260 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timezone
from typing import Callable
from redis import Redis
from .constants import (
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT,
MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT,
MOBILEDE_REFRESH_CYCLE_ID_KEY,
MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
)
def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str:
return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id)
def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
def _mobilede_redis_text(value: object) -> str:
if isinstance(value, bytes):
return value.decode("utf-8", errors="ignore")
return str(value or "")
def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
return bool(
redis_client.set(
_mobilede_refresh_cycle_finalized_key(cycle_id),
"1",
nx=True,
ex=ttl,
)
)
def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
keys = [
MOBILEDE_REFRESH_CYCLE_ID_KEY,
MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
]
if cycle_id:
keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
redis_client.delete(*keys)
def _mobilede_start_refresh_cycle(
redis_client: Redis,
*,
total_segments: int,
logger: logging.Logger,
) -> str:
cycle_id = uuid.uuid4().hex[:16]
started_at = datetime.now(timezone.utc).isoformat()
total = max(0, int(total_segments))
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
pipe = redis_client.pipeline()
pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
pipe.delete(done_set_key)
pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
pipe.execute()
logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
return cycle_id
def _mobilede_get_or_start_refresh_cycle(
redis_client: Redis,
*,
total_segments: int,
logger: logging.Logger,
) -> str:
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id:
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0)
if total_now <= 0 or done_now < total_now:
return active_cycle_id
return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger)
def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None:
if not cycle_id:
return None
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return None
started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
if not started_at_raw:
return None
try:
seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
if seen_at.tzinfo is None:
seen_at = seen_at.replace(tzinfo=timezone.utc)
return seen_at
except Exception:
logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw)
return None
def _mobilede_refresh_cycle_progress(
redis_client: Redis,
*,
cycle_id: str | None,
total_segments_hint: int = 0,
) -> tuple[int, int, int]:
if not cycle_id:
return 0, max(0, int(total_segments_hint)), 0
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return 0, max(0, int(total_segments_hint)), 0
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
if total_now <= 0:
total_now = max(done_now, int(total_segments_hint or 0))
left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0
return done_now, total_now, left_now
def _mobilede_track_refresh_cycle_segment(
redis_client: Redis,
*,
cycle_id: str | None,
segment: dict[str, object] | None,
total_segments_hint: int,
segment_fingerprint: Callable[[dict[str, object] | None], str],
) -> tuple[int, int, bool]:
if not cycle_id or not segment:
return 0, max(0, int(total_segments_hint)), False
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return 0, 0, False
done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0:
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
return done_now, total_now, False
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
redis_client.expire(done_set_key, ttl)
done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY))
redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
if total_now <= 0:
total_now = max(done_now, int(total_segments_hint or 0))
redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl)
if total_now > 0 and done_now > total_now:
done_now = total_now
redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl)
should_finalize = False
if total_now > 0 and done_now >= total_now:
should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
return done_now, total_now, should_finalize
def _mobilede_finalize_refresh_cycle_sold_marking(
redis_client: Redis,
*,
cycle_id: str,
logger: logging.Logger,
get_persistence: Callable[[], object],
origin_prefixes: tuple[str, ...],
post_refresh_probe_enabled: bool,
post_refresh_probe_batch_size: int,
schedule_post_refresh_probe: Callable[[], None],
) -> int:
started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
if not started_at_raw:
logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
return 0
try:
cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
if cutoff.tzinfo is None:
cutoff = cutoff.replace(tzinfo=timezone.utc)
except Exception:
logger.warning(
"mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s",
started_at_raw,
cycle_id,
)
return 0
sold_marked = get_persistence().mark_sold_not_seen_since(
cutoff,
prefix=origin_prefixes,
safety_ratio=0.8,
)
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
logger.info(
"mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
cycle_id,
done_now,
total_now,
cutoff.isoformat(),
sold_marked,
)
if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
try:
schedule_post_refresh_probe()
except Exception:
logger.warning(
"mobile.de post-refresh sold probe scheduling failed: cycle=%s",
cycle_id,
exc_info=True,
)
return sold_marked
def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
redis_client: Redis,
*,
cycle_id: str | None,
total_segments_hint: int = 0,
logger: logging.Logger,
finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int],
) -> bool:
if not cycle_id:
return False
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return False
done_now, total_now, _left_now = _mobilede_refresh_cycle_progress(
redis_client,
cycle_id=cycle_id,
total_segments_hint=total_segments_hint,
)
if total_now <= 0:
return False
if done_now < total_now:
redis_client.set(
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
str(total_now),
ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
)
logger.warning(
"mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s",
cycle_id,
done_now,
total_now,
)
if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
return False
finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)
return True

1115
mobilede_scraper/worker/search_sync.py Normal file
View File

File diff suppressed because it is too large Load Diff

2993
mobilede_scraper/worker/tasks.py
View File

File diff suppressed because it is too large Load Diff

240
tests/test_worker_runtime_tasks.py
View File

@@ -1,7 +1,8 @@
from __future__ import annotations
from datetime import datetime, timezone
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from mobilede_scraper.worker import tasks
@@ -83,6 +84,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(key1, key2)
self.assertEqual(len(key1), 16)
def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
parent = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
"make_id": "3500",
"price_min": "1",
"price_max": "5000",
"year_max": "2004",
"mileage_max": "100000",
}
parent_key = tasks._mobilede_segment_fingerprint(parent)
child_a = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
"mileage_max": "50000",
"overflow_parent": parent_key,
}
child_b = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
"mileage_min": "50001",
"mileage_max": "100000",
"overflow_parent": parent_key,
}
pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
self.assertEqual(pruned, [child_a, child_b])
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
@@ -102,86 +131,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
def test_preplan_splits_dense_segment_before_queueing(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertEqual(len(planned), 2)
self.assertTrue(all(item.get("total_results") == 100 for item in planned))
self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
def test_preplan_has_hard_segment_limit(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 5000
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
{**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
{**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertGreaterEqual(len(planned), 1)
self.assertLessEqual(len(planned), 3)
def test_preplan_probe_budget_is_global(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
@@ -215,30 +164,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(calls["count"], 2)
self.assertEqual(len(planned), 5)
def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
try:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
children = tasks._mobilede_build_overflow_child_segments(
segment={
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
"price_min": "5001",
"price_max": "10000",
"year_min": "2005",
"year_max": "2009",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
},
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
finally:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
self.assertEqual(len(children), 2)
self.assertEqual(children[0].get("mileage_max"), "150000")
self.assertEqual(children[1].get("mileage_min"), "150001")
self.assertIsNone(children[1].get("mileage_max"))
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
@@ -278,6 +203,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
redis_client = self._FakeRedis()
started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
persistence = MagicMock()
persistence.mark_sold_not_seen_since.return_value = 42
with (
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
):
finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
redis_client,
cycle_id="cycle-lag",
total_segments_hint=377,
)
self.assertTrue(finalized)
self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
persistence.mark_sold_not_seen_since.assert_called_once_with(
started_at,
prefix=("mobile.de:", "mobilede:"),
safety_ratio=0.8,
)
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
@@ -357,9 +310,32 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(done, 0)
self.assertEqual(total, 2)
self.assertEqual(total, 0)
self.assertFalse(should_finalize)
def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
segment = {
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
"total_results": 995,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
added = tasks._mobilede_try_expand_overflow_segment(
redis_client,
settings,
segment=segment,
listing_count=1000,
unique_count=995,
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
self.assertEqual(added, 0)
redis_client.sadd.assert_not_called()
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
@@ -390,58 +366,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
search_url = (
"https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
"?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
)
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
segments = tasks._expand_mobilede_search_url_segment(
{
"label": "Cars",
"search_url": search_url,
"start_page": 1,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
make_counts: dict[str, int] = {}
for segment in segments:
make_id = str(segment.get("make_id") or "")
make_counts[make_id] = make_counts.get(make_id, 0) + 1
self.assertIsNone(segment.get("mileage_min"))
self.assertIsNone(segment.get("mileage_max"))
self.assertEqual(make_counts, {"3500": 59, "11000": 59})
self.assertEqual(len(segments), 118)
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
]
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertEqual(len(segments), 118)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
import os