qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve runtime sync flow

Browse Source
This commit is contained in:
qananasikq
2026-05-11 12:45:51 +03:00
parent 124dfc514f
commit d9d073641b
13 changed files with 1224 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
mobilede_scraper/worker/planner.py
View File

@@ -1,9 +1,24 @@
from __future__ import annotations
# Segment planning and overflow splitting live here to keep tasks.py focused
# on Celery orchestration. The module intentionally reuses task-layer
# helpers and globals that are synchronized from tasks.py before calls.
# Планирование сегментов вынесено сюда, чтобы `tasks.py` оставался короче.
# Модуль использует хелперы из `tasks.py`.
from .tasks import * # noqa: F401,F403
from .tasks import (
_mobilede_make_segment_url,
_mobilede_segment_fingerprint,
_mobilede_segment_key,
_mobilede_segment_label,
_mobilede_short_segment_label,
_mobilede_short_segment_ref,
_mobilede_should_skip_dynamic_segment,
_mobilede_should_skip_planned_segment,
_mobilede_site_make_options_cache,
_mobilede_try_mark_bootstrap_segment_dispatched,
_mobilede_url_query_values,
)
_MOBILEDE_RUNTIME_PLAN_VERSION = 2
def _mobilede_price_ranges() -> list[tuple[int, int | None]]:
@@ -40,10 +55,10 @@ def _mobilede_price_ranges() -> list[tuple[int, int | None]]:
def _mobilede_price_ranges_for_segment(segment: dict[str, object] | None = None) -> list[tuple[int, int | None]]:
make_id = str((segment or {}).get("make_id") or "").strip()
if make_id == "3500" and MOBILEDE_COMPACT_SEGMENTS:
# Используем границы, близкие к самим фильтрам mobile.de, чтобы BMW
# сразу попадали в более плотные корзины и реже дорезались в хвосте.
del segment
if MOBILEDE_COMPACT_SEGMENTS and os.getenv("MOBILEDE_DENSE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}:
# Плотные корзины включаются для всех марок из пользовательской ссылки.
# Так покрытие не зависит от конкретных make_id и не теряет хвосты за лимитом 50 страниц.
return [
(1, 5000),
(5001, 10000),
@@ -63,6 +78,14 @@ def _mobilede_price_ranges_for_segment(segment: dict[str, object] | None = None)
return _mobilede_price_ranges()
def _mobilede_filtered_url_uses_adaptive_plan() -> bool:
return os.getenv("MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN", "false").strip().lower() in {"1", "true", "yes", "on"}
def _mobilede_skip_late_overflow_children_during_bootstrap() -> bool:
return os.getenv("MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"}
def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]:
if MOBILEDE_COMPACT_SEGMENTS:
return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)]
@@ -93,8 +116,8 @@ def _mobilede_year_ranges_for_segment_price(
price_min: int,
price_max: int | None,
) -> list[tuple[int | None, int | None]]:
make_id = str((segment or {}).get("make_id") or "").strip()
if make_id != "3500":
del segment
if not (MOBILEDE_COMPACT_SEGMENTS and os.getenv("MOBILEDE_DENSE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}):
return _mobilede_year_ranges_for_price(price_min, price_max)
upper_bound = int(price_max) if price_max is not None else int(price_min)
@@ -107,6 +130,35 @@ def _mobilede_year_ranges_for_segment_price(
return _mobilede_year_ranges_for_price(price_min, price_max)
def _mobilede_interleave_segments_by_make(segments: list[dict[str, object]]) -> list[dict[str, object]]:
"""Mix make blocks so an early bootstrap pass covers the whole source URL."""
groups: dict[str, list[dict[str, object]]] = {}
order: list[str] = []
passthrough: list[dict[str, object]] = []
for segment in segments:
make_id = str(segment.get("make_id") or "").strip()
if not make_id:
passthrough.append(segment)
continue
if make_id not in groups:
groups[make_id] = []
order.append(make_id)
groups[make_id].append(segment)
if len(order) <= 1:
return segments
mixed: list[dict[str, object]] = []
max_len = max(len(items) for items in groups.values())
for index in range(max_len):
for make_id in order:
items = groups[make_id]
if index < len(items):
mixed.append(items[index])
mixed.extend(passthrough)
return mixed
def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]:
if MOBILEDE_COMPACT_SEGMENTS:
return [(None, 100000), (100001, 200000), (200001, None)]
@@ -125,8 +177,7 @@ def _mobilede_should_pre_split_mileage(
if MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:
return True
# Restore the older balanced plan: split mileage only in buckets that are
# usually dense enough to hit mobile.de's 50-page cap.
# Делим по пробегу только плотные корзины.
if price_max is not None and price_max <= MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX:
return year_max is not None and year_max <= 2009
@@ -310,7 +361,7 @@ def _mobilede_load_learned_runtime_segments(source_segments: list[dict[str, obje
payload = json.load(fh)
if not isinstance(payload, dict):
return None
if payload.get("plan_version") != 1:
if payload.get("plan_version") != _MOBILEDE_RUNTIME_PLAN_VERSION:
return None
expected = _mobilede_segments_source_fingerprint(source_segments)
if str(payload.get("source_fingerprint") or "") != expected:
@@ -346,7 +397,7 @@ def _mobilede_save_learned_runtime_segments(
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
pruned_segments = _mobilede_prune_overflow_parent_segments(runtime_segments)
payload = {
"plan_version": 1,
"plan_version": _MOBILEDE_RUNTIME_PLAN_VERSION,
"source_fingerprint": _mobilede_segments_source_fingerprint(source_segments),
"updated_at": datetime.now(timezone.utc).isoformat(),
"segments": pruned_segments,
@@ -602,16 +653,36 @@ def _mobilede_split_price_ranges_for_overflow(
left = int(price_min or 1)
right = price_max
if right is None:
step = max(5000, min(50000, left))
pivot = left + step
return [(left, pivot), (pivot + 1, None)]
child_budget = max(2, int(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS))
step = max(MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN, 5000, min(50000, left))
ranges: list[tuple[int, int | None]] = []
current = left
for _index in range(child_budget - 1):
upper = current + step
ranges.append((current, upper))
current = upper + 1
step = min(step * 2, 100000)
ranges.append((current, None))
return ranges
span = int(right) - int(left)
if span < MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN:
return []
pivot = int(left) + span // 2
if pivot <= int(left) or pivot >= int(right):
return []
return [(int(left), pivot), (pivot + 1, int(right))]
child_count = min(
max(2, int(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS)),
max(2, span // MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN + 1),
)
step = max(1, span // child_count)
ranges = []
current = int(left)
for index in range(child_count):
upper = int(right) if index == child_count - 1 else min(int(right), current + step)
if upper < current:
break
ranges.append((current, upper))
current = upper + 1
if current > int(right):
break
return ranges
def _mobilede_split_mileage_ranges_for_overflow(
@@ -632,9 +703,16 @@ def _mobilede_split_mileage_ranges_for_overflow(
return [(None, pivot), (pivot + 1, right)]
if mileage_min is not None and mileage_max is None:
left = int(mileage_min)
child_budget = max(2, int(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS))
step = max(25000, min(100000, left))
pivot = left + step
return [(left, pivot), (pivot + 1, None)]
ranges: list[tuple[int | None, int | None]] = []
current = left
for _index in range(child_budget - 1):
upper = current + step
ranges.append((current, upper))
current = upper + 1
ranges.append((current, None))
return ranges
assert mileage_min is not None and mileage_max is not None
left = int(mileage_min)
@@ -642,10 +720,19 @@ def _mobilede_split_mileage_ranges_for_overflow(
span = right - left
if span < 10000:
return _mobilede_split_fine_mileage_ranges(left, right)
pivot = left + span // 2
if pivot <= left or pivot >= right:
return []
return [(left, pivot), (pivot + 1, right)]
child_count = min(max(2, int(MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS)), max(2, span // 10000 + 1))
step = max(1, span // child_count)
ranges = []
current = left
for index in range(child_count):
upper = right if index == child_count - 1 else min(right, current + step)
if upper < current:
break
ranges.append((current, upper))
current = upper + 1
if current > right:
break
return ranges
def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[int | None, int | None]]:
@@ -659,6 +746,15 @@ def _mobilede_root_mileage_ranges_for_overflow(max_children: int) -> list[tuple[
pivot = 150000 if MOBILEDE_COMPACT_SEGMENTS else 100000
low_cap = 75000 if MOBILEDE_COMPACT_SEGMENTS else 50000
return [(None, low_cap), (low_cap + 1, pivot), (pivot + 1, None)]
if MOBILEDE_COMPACT_SEGMENTS:
ranges: list[tuple[int | None, int | None]] = [
(None, 50000),
(50001, 100000),
(100001, 150000),
(150001, 200000),
(200001, None),
]
return ranges[:child_budget]
return _mobilede_mileage_ranges()
@@ -694,7 +790,7 @@ def _mobilede_range_span(value_min: int | None, value_max: int | None) -> int |
return max(0, int(value_max) - int(value_min))
def _mobilede_should_avoid_bmw_mileage_split(
def _mobilede_should_avoid_tiny_mileage_split(
*,
depth: int,
price_min: int | None,
@@ -809,11 +905,10 @@ def _mobilede_build_overflow_candidate_groups(
year_max = _mobilede_parse_optional_int(segment.get("year_max"))
price_min = _mobilede_parse_optional_int(segment.get("price_min"))
price_max = _mobilede_parse_optional_int(segment.get("price_max"))
make_id = str(segment.get("make_id") or "").strip()
previous_split_kind = str(segment.get("overflow_split") or "").strip()
mileage_splits = _mobilede_split_mileage_ranges_for_overflow(mileage_min, mileage_max)
if make_id == "3500" and mileage_splits:
if _mobilede_should_avoid_bmw_mileage_split(
if mileage_splits:
if _mobilede_should_avoid_tiny_mileage_split(
depth=depth,
price_min=price_min,
price_max=price_max,
@@ -844,7 +939,7 @@ def _mobilede_build_overflow_candidate_groups(
price_splits = _mobilede_split_price_ranges_for_overflow(price_min, price_max)
if price_splits:
child_segments = []
for child_price_min, child_price_max in price_splits[:2]:
for child_price_min, child_price_max in price_splits[:MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS]:
child_segments.append(
_mobilede_make_overflow_child_segment(
segment,
@@ -869,8 +964,8 @@ def _mobilede_build_overflow_candidate_groups(
previous_split_kind == "year"
and depth >= MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH
)
if allow_year_split and make_id == "3500":
# Для BMW не уходим в слишком узкие year-ветки.
if allow_year_split:
# Не уходим в слишком узкие year-ветки, если уже есть более полезная разбивка.
if year_span is not None and year_span < max(4, MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN):
allow_year_split = False
elif price_splits or mileage_splits:
@@ -879,7 +974,7 @@ def _mobilede_build_overflow_candidate_groups(
year_splits = _mobilede_split_year_ranges_for_overflow(year_min, year_max) if allow_year_split else []
if year_splits:
child_segments = []
for child_year_min, child_year_max in year_splits[:2]:
for child_year_min, child_year_max in year_splits[:MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS]:
child_segments.append(
_mobilede_make_overflow_child_segment(
segment,
@@ -897,7 +992,7 @@ def _mobilede_build_overflow_candidate_groups(
if mileage_splits:
child_segments = []
for child_mileage_min, child_mileage_max in mileage_splits[:2]:
for child_mileage_min, child_mileage_max in mileage_splits[:MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS]:
child_segments.append(
_mobilede_make_overflow_child_segment(
segment,
@@ -913,10 +1008,7 @@ def _mobilede_build_overflow_candidate_groups(
)
_mobilede_append_overflow_candidate_group(candidate_groups, "mileage", child_segments)
if make_id == "11000":
candidate_groups.sort(key=lambda item: {"price": 0, "recent_price": 0, "year": 1, "root_mileage": 2, "mileage": 3}.get(item[0], 9))
elif make_id == "3500":
candidate_groups.sort(key=lambda item: {"recent_price": 0, "price": 1, "year": 2, "mileage": 3, "root_mileage": 4}.get(item[0], 9))
candidate_groups.sort(key=lambda item: {"recent_price": 0, "price": 1, "year": 2, "mileage": 3, "root_mileage": 4}.get(item[0], 9))
return candidate_groups
@@ -1003,17 +1095,16 @@ def _mobilede_score_overflow_candidate_group(
split_bias += depth * max(1, MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY // 2)
if kind == "mileage" and previous_split_kind == "mileage":
split_bias += MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if make_id == "3500":
if price_span is not None and price_span <= 10000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if price_span is not None and price_span <= 5000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if mileage_span is not None and mileage_span <= 100000:
split_bias += max(1, MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY // 2)
if mileage_span is not None and mileage_span <= 50000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if mileage_span is not None and mileage_span <= 25000:
split_bias += MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if price_span is not None and price_span <= 10000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if price_span is not None and price_span <= 5000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if mileage_span is not None and mileage_span <= 100000:
split_bias += max(1, MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY // 2)
if mileage_span is not None and mileage_span <= 50000:
split_bias += MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
if mileage_span is not None and mileage_span <= 25000:
split_bias += MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY + MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY
distance_to_target += micro_children * max(1, MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY // 2)
@@ -1050,10 +1141,12 @@ def _mobilede_overflow_candidate_group_is_useful(
return True
useful_children = sum(total >= useful_floor for total in child_totals)
non_empty_children = sum(total > 0 for total in child_totals)
micro_children = sum(total < lower_target for total in child_totals)
tiny_children = sum(total < tiny_threshold for total in child_totals)
max_child_total = max(child_totals)
make_id = str(segment.get("make_id") or "").strip()
sum_child_total = sum(child_totals)
parent_total = _mobilede_segment_total_results(segment)
depth = max(0, int(_mobilede_parse_optional_int(segment.get("overflow_depth")) or 0))
previous_split_kind = str(segment.get("overflow_split") or "").strip()
price_span = _mobilede_range_span(
@@ -1065,23 +1158,28 @@ def _mobilede_overflow_candidate_group_is_useful(
_mobilede_parse_optional_int(segment.get("mileage_max")),
)
if parent_total is not None and int(parent_total) > target:
if non_empty_children > 0 and sum_child_total >= int(parent_total * 0.5):
return True
if max_child_total >= lower_target:
return True
if tiny_children >= len(child_totals):
return False
if micro_children > MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN and useful_children == 0:
return False
if make_id == "3500":
if kind in {"mileage", "root_mileage"}:
if previous_split_kind == "mileage" and useful_children == 0:
return False
if depth >= 2 and useful_children == 0:
return False
if price_span is not None and price_span <= 10000 and useful_children == 0:
return False
if mileage_span is not None and mileage_span <= 100000 and micro_children > 0 and useful_children == 0:
return False
if kind == "year" and useful_children == 0 and max_child_total < lower_target:
if kind in {"mileage", "root_mileage"}:
if previous_split_kind == "mileage" and useful_children == 0:
return False
if depth >= 2 and useful_children == 0:
return False
if price_span is not None and price_span <= 10000 and useful_children == 0:
return False
if mileage_span is not None and mileage_span <= 100000 and micro_children > 0 and useful_children == 0:
return False
if kind == "year" and useful_children == 0 and max_child_total < lower_target:
return False
return useful_children > 0 or max_child_total >= lower_target
@@ -1164,8 +1262,8 @@ def _mobilede_finalize_preplanned_segment(segment: dict[str, object], total_resu
item["total_results"] = total_results
if total_results is not None:
label = str(item.get("label") or _mobilede_segment_key(item))
if "total=" not in label:
item["label"] = f"{label} | total={total_results}"
label = re.sub(r"\s*\|\s*total=\d+", "", label)
item["label"] = f"{label} | total={total_results}"
return item
@@ -1464,6 +1562,12 @@ def _queue_mobilede_overflow_child_segments(
) -> int:
if not parent_segment:
return 0
if bootstrap_run and only_new is not True and _mobilede_skip_late_overflow_children_during_bootstrap():
logger.info(
"mobile.de late overflow children not queued during bootstrap: parent=%s reason=preplan_first_pass",
_mobilede_short_segment_label(parent_segment),
)
return 0
parent_fingerprint = _mobilede_segment_fingerprint(parent_segment)
children = _mobilede_get_overflow_child_segments(
redis_client,
@@ -1529,7 +1633,14 @@ def _queue_mobilede_overflow_child_segments(
def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int | None:
try:
client = MobileDeClient.for_worker(delay_seconds=0)
page = client.fetch_search_page(page_number=1, search_url=search_url, **params)
probe_timeout = max(5, int(os.getenv("MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS", "12")))
page = client.fetch_search_page(
page_number=1,
search_url=search_url,
timeout=probe_timeout,
max_retries=0,
**params,
)
return int(page.total_results or 0)
except Exception as exc:
logger.warning("mobile.de segment probe failed: params=%s error=%s", params, exc)
@@ -1662,7 +1773,7 @@ def _mobilede_split_search_url_segment_by_make(segment: dict[str, object]) -> li
def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) -> list[dict[str, object]] | None:
if not MOBILEDE_PREPLAN_SEGMENT_PROBES:
if not MOBILEDE_PREPLAN_SEGMENT_PROBES or not _mobilede_filtered_url_uses_adaptive_plan():
return None
search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip()
@@ -1674,11 +1785,15 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
planned: list[dict[str, object]] = []
probes_used = 0
probe_limit = max(1, int(MOBILEDE_PREPLAN_MAX_PROBES))
started_at = time.monotonic()
max_seconds = int(MOBILEDE_ADAPTIVE_URL_MAX_SECONDS)
def _probe(params: dict[str, str | int | None]) -> int | None:
nonlocal probes_used
if probes_used >= probe_limit:
return None
if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
return None
probes_used += 1
if probes_used == 1 or probes_used % 25 == 0:
_mobilede_touch_planning_progress("adaptive_url_planning")
@@ -1692,7 +1807,11 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
)
return _mobilede_probe_total(search_url, **params)
time_budget_reached = False
for price_min, price_max in _mobilede_price_ranges_for_segment(segment):
if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
time_budget_reached = True
break
price_params: dict[str, str | int | None] = {"p": _mobilede_range_value(price_min, price_max)}
price_label = _mobilede_price_label(price_min, price_max)
price_total = _probe(price_params)
@@ -1714,8 +1833,14 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
continue
for year_min, year_max in _mobilede_year_ranges_for_segment_price(segment, price_min, price_max):
if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
time_budget_reached = True
break
year_label = _mobilede_year_label(year_min, year_max)
for refined_price_min, refined_price_max in _mobilede_price_subranges_for_hot_year(price_min, price_max, year_min, year_max):
if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
time_budget_reached = True
break
refined_price_label = _mobilede_price_label(refined_price_min, refined_price_max)
year_params = {
"p": _mobilede_range_value(refined_price_min, refined_price_max),
@@ -1741,6 +1866,9 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
continue
for mileage_min, mileage_max in _mobilede_mileage_ranges():
if max_seconds > 0 and time.monotonic() - started_at >= max_seconds:
time_budget_reached = True
break
mileage_params = dict(year_params)
mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max)
mileage_label = _mobilede_mileage_label(mileage_min, mileage_max)
@@ -1761,6 +1889,20 @@ def _expand_mobilede_search_url_segment_by_probe(segment: dict[str, object]) ->
mileage_range=(mileage_min, mileage_max),
)
)
if time_budget_reached:
break
if time_budget_reached:
break
if time_budget_reached:
logger.warning(
"mobile.de adaptive URL planning time budget reached: base=%s seconds=%s segments=%s probes=%s/%s",
_mobilede_short_segment_label(segment),
max_seconds,
len(planned),
probes_used,
probe_limit,
)
logger.info(
"mobile.de adaptive URL segments planned: base=%s segments=%s probes=%s limit=%s",
@@ -1874,6 +2016,33 @@ def _mobilede_refine_dense_planned_segments(segments: list[dict[str, object]]) -
return refined
def _mobilede_refine_dense_planned_segments_until_stable(segments: list[dict[str, object]]) -> list[dict[str, object]]:
max_passes = max(1, int(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES", "2")))
refined = [dict(item) for item in segments]
for pass_index in range(1, max_passes + 1):
before = len(refined)
refined = _mobilede_refine_dense_planned_segments(refined)
dense_count = sum(
1
for item in refined
if (_mobilede_segment_total_results(item) or 0) > min(
MOBILEDE_SEGMENT_TARGET_RESULTS,
_mobilede_overflow_threshold(MOBILEDE_MAX_PAGE_NUMBER),
)
)
logger.info(
"mobile.de dense refine pass complete: pass=%s/%s before=%s after=%s dense_left=%s",
pass_index,
max_passes,
before,
len(refined),
dense_count,
)
if dense_count <= 0 or len(refined) >= MOBILEDE_PREPLAN_MAX_SEGMENTS:
break
return refined
def _mobilede_should_refine_adaptive_segments() -> bool:
return os.getenv("MOBILEDE_REFINE_ADAPTIVE_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
@@ -2079,17 +2248,15 @@ def _expand_mobilede_search_url_segment(
def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]:
env_search_urls = settings.listing.filtered_search_urls
# Ready-made filtered URLs default to the old deterministic price/year split:
# it starts immediately and is easier to keep stable. Probe-based adaptive
# planning can still be enabled explicitly when we need denser pre-plans.
# Готовые фильтрованные URL по умолчанию используют старое
# детерминированное деление по цене/году: оно стартует сразу и его проще
# держать стабильным. Адаптивное планирование на основе probe можно
# включить явно, когда нужен более плотный pre-plan.
fast_start_filtered_urls = (
bool(env_search_urls)
and os.getenv("MOBILEDE_FILTERED_URL_FAST_START", "false").strip().lower() in {"1", "true", "yes", "on"}
)
adaptive_filtered_urls = (
bool(env_search_urls)
and os.getenv("MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN", "false").strip().lower() in {"1", "true", "yes", "on"}
)
adaptive_filtered_urls = bool(env_search_urls) and _mobilede_filtered_url_uses_adaptive_plan()
if env_search_urls:
segments = _mobilede_source_segments_from_settings(settings)
if any(str(item.get("runtime_brand") or "").strip() for item in segments):
@@ -2104,7 +2271,7 @@ def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, objec
segments = _mobilede_source_segments_from_settings(settings)
learned_segments = _mobilede_load_learned_runtime_segments(segments)
if learned_segments is not None:
return learned_segments
return _mobilede_interleave_segments_by_make(learned_segments)
expanded: list[dict[str, object]] = []
expanded_from_adaptive_url = False
for segment in segments:
@@ -2121,6 +2288,7 @@ def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, objec
expanded_from_adaptive_url = True
if len(expanded) != len(segments):
logger.info("mobile.de segments planned: input=%s total=%s", len(segments), len(expanded))
expanded = _mobilede_interleave_segments_by_make(expanded)
if fast_start_filtered_urls:
logger.info(
"mobile.de fast-start runtime segments ready: input_urls=%s final=%s reason=filtered_search_urls",
@@ -2130,7 +2298,7 @@ def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, objec
return [_mobilede_finalize_preplanned_segment(item, item.get("total_results")) for item in expanded]
if expanded_from_adaptive_url:
if _mobilede_should_refine_adaptive_segments():
refined = _mobilede_refine_dense_planned_segments(expanded)
refined = _mobilede_refine_dense_planned_segments_until_stable(expanded)
logger.info(
"mobile.de adaptive URL plan refined: before=%s after=%s reason=dense_segments",
len(expanded),